[ { "50.00 percentile latency (ns)": 7272980, "90.00 percentile latency (ns)": 9629055, "90th percentile latency (ns)": 9629055, "95.00 percentile latency (ns)": 11572662, "97.00 percentile latency (ns)": 12899029, "99.00 percentile latency (ns)": 15576216, "99.90 percentile latency (ns)": 32024463, "Max latency (ns)": 74455334, "Mean latency (ns)": 7940213, "Min duration satisfied": "Yes", "Min latency (ns)": 6216617, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 125.63, "QPS w/o loadgen overhead": 125.94, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.242, "characteristics.good": 34121, "characteristics.power": 25.907183333333336, "characteristics.power.normalized_per_core": 3.238397916666667, "characteristics.power.normalized_per_processor": 25.907183333333336, "characteristics.samples_per_second": 125.94, "characteristics.samples_per_second.normalized_per_core": 15.7425, "characteristics.samples_per_second.normalized_per_processor": 125.94, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "e74684940329541c", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7911166, "90.00 percentile latency (ns)": 10938850, "90th percentile latency (ns)": 10938850, "95.00 percentile latency (ns)": 12975998, "97.00 percentile latency (ns)": 14286099, "99.00 percentile latency (ns)": 17023210, "99.90 percentile latency (ns)": 35423488, "Max latency (ns)": 160785610, "Mean latency (ns)": 8640778, "Min duration satisfied": "Yes", "Min latency (ns)": 6641026, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 115.51, "QPS w/o loadgen overhead": 115.73, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.738, "characteristics.good": 35369, "characteristics.power": 25.49238333333335, "characteristics.power.normalized_per_core": 3.186547916666669, "characteristics.power.normalized_per_processor": 25.49238333333335, "characteristics.samples_per_second": 115.73, "characteristics.samples_per_second.normalized_per_core": 14.46625, "characteristics.samples_per_second.normalized_per_processor": 115.73, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "4bfa94c47ae0ae8e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 78625979, "90.00 percentile latency (ns)": 89772868, "90th percentile latency (ns)": 89772868, "95.00 percentile latency (ns)": 94199691, "97.00 percentile latency (ns)": 96538418, "99.00 percentile latency (ns)": 104299886, "99.90 percentile latency (ns)": 178814199, "Max latency (ns)": 192755891, "Mean latency (ns)": 79720976, "Min duration satisfied": "Yes", "Min latency (ns)": 67947737, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 12.54, "QPS w/o loadgen overhead": 12.54, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 78.514, "characteristics.good": 39257, "characteristics.power": 21.206216666666666, "characteristics.power.normalized_per_core": 2.6507770833333333, "characteristics.power.normalized_per_processor": 21.206216666666666, "characteristics.samples_per_second": 12.54, "characteristics.samples_per_second.normalized_per_core": 1.5675, "characteristics.samples_per_second.normalized_per_processor": 12.54, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite4-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.6582, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "44964a79f5dbb0a0", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6993247, "90.00 percentile latency (ns)": 9910143, "90th percentile latency (ns)": 9910143, "95.00 percentile latency (ns)": 12315725, "97.00 percentile latency (ns)": 13545418, "99.00 percentile latency (ns)": 15563205, "99.90 percentile latency (ns)": 30036056, "Max latency (ns)": 122747374, "Mean latency (ns)": 7726643, "Min duration satisfied": "Yes", "Min latency (ns)": 5955504, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 129.28, "QPS w/o loadgen overhead": 129.42, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.542, "characteristics.good": 31771, "characteristics.power": 20.403399999999973, "characteristics.power.normalized_per_core": 2.5504249999999966, "characteristics.power.normalized_per_processor": 20.403399999999973, "characteristics.samples_per_second": 129.42, "characteristics.samples_per_second.normalized_per_core": 16.1775, "characteristics.samples_per_second.normalized_per_processor": 129.42, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "9745478c2d52f779", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2265351, "90.00 percentile latency (ns)": 3374649, "90th percentile latency (ns)": 3374649, "95.00 percentile latency (ns)": 4198015, "97.00 percentile latency (ns)": 4574064, "99.00 percentile latency (ns)": 6696975, "99.90 percentile latency (ns)": 20101170, "Max latency (ns)": 135472817, "Mean latency (ns)": 2591644, "Min duration satisfied": "Yes", "Min latency (ns)": 1831443, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 384.01, "QPS w/o loadgen overhead": 385.86, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 52.172, "characteristics.good": 26086, "characteristics.power": 22.701571666666652, "characteristics.power.normalized_per_core": 2.8376964583333315, "characteristics.power.normalized_per_processor": 22.701571666666652, "characteristics.samples_per_second": 385.86, "characteristics.samples_per_second.normalized_per_core": 48.2325, "characteristics.samples_per_second.normalized_per_processor": 385.86, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "787834925bf5f2b4", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2468086, "90.00 percentile latency (ns)": 3988450, "90th percentile latency (ns)": 3988450, "95.00 percentile latency (ns)": 4715488, "97.00 percentile latency (ns)": 5410020, "99.00 percentile latency (ns)": 8200873, "99.90 percentile latency (ns)": 22693278, "Max latency (ns)": 123334660, "Mean latency (ns)": 2887360, "Min duration satisfied": "Yes", "Min latency (ns)": 2010655, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 344.83, "QPS w/o loadgen overhead": 346.34, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.402, "characteristics.good": 29201, "characteristics.power": 23.164531666666676, "characteristics.power.normalized_per_core": 2.8955664583333345, "characteristics.power.normalized_per_processor": 23.164531666666676, "characteristics.samples_per_second": 346.34, "characteristics.samples_per_second.normalized_per_core": 43.2925, "characteristics.samples_per_second.normalized_per_processor": 346.34, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "da0bf4a4fbc4eb30", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4681071, "90.00 percentile latency (ns)": 6842036, "90th percentile latency (ns)": 6842036, "95.00 percentile latency (ns)": 8190831, "97.00 percentile latency (ns)": 9317791, "99.00 percentile latency (ns)": 11976858, "99.90 percentile latency (ns)": 28056652, "Max latency (ns)": 127344079, "Mean latency (ns)": 5325011, "Min duration satisfied": "Yes", "Min latency (ns)": 3852844, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 187.27, "QPS w/o loadgen overhead": 187.79, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 60.976, "characteristics.good": 30488, "characteristics.power": 23.66138333333334, "characteristics.power.normalized_per_core": 2.9576729166666675, "characteristics.power.normalized_per_processor": 23.66138333333334, "characteristics.samples_per_second": 187.79, "characteristics.samples_per_second.normalized_per_core": 23.47375, "characteristics.samples_per_second.normalized_per_processor": 187.79, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "94ce7b29cefe8ba6", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2589166, "90.00 percentile latency (ns)": 3689949, "90th percentile latency (ns)": 3689949, "95.00 percentile latency (ns)": 4333752, "97.00 percentile latency (ns)": 4897362, "99.00 percentile latency (ns)": 7005770, "99.90 percentile latency (ns)": 18754400, "Max latency (ns)": 63487856, "Mean latency (ns)": 2892981, "Min duration satisfied": "Yes", "Min latency (ns)": 2138811, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 344.99, "QPS w/o loadgen overhead": 345.66, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 48.162, "characteristics.good": 24081, "characteristics.power": 20.118286666666663, "characteristics.power.normalized_per_core": 2.514785833333333, "characteristics.power.normalized_per_processor": 20.118286666666663, "characteristics.samples_per_second": 345.66, "characteristics.samples_per_second.normalized_per_core": 43.2075, "characteristics.samples_per_second.normalized_per_processor": 345.66, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "374a408e734d5ace", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 13188553, "90.00 percentile latency (ns)": 19281168, "90th percentile latency (ns)": 19281168, "95.00 percentile latency (ns)": 20520711, "97.00 percentile latency (ns)": 21645729, "99.00 percentile latency (ns)": 26984860, "99.90 percentile latency (ns)": 46796519, "Max latency (ns)": 133426846, "Mean latency (ns)": 14663460, "Min duration satisfied": "Yes", "Min latency (ns)": 10358851, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 68.1, "QPS w/o loadgen overhead": 68.2, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.068, "characteristics.good": 37534, "characteristics.power": 26.263649999999995, "characteristics.power.normalized_per_core": 3.2829562499999994, "characteristics.power.normalized_per_processor": 26.263649999999995, "characteristics.samples_per_second": 68.2, "characteristics.samples_per_second.normalized_per_core": 8.525, "characteristics.samples_per_second.normalized_per_processor": 68.2, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite0-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "6b65e09c40e68c7f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3002217, "90.00 percentile latency (ns)": 4357998, "90th percentile latency (ns)": 4357998, "95.00 percentile latency (ns)": 5207981, "97.00 percentile latency (ns)": 6082756, "99.00 percentile latency (ns)": 8310956, "99.90 percentile latency (ns)": 21694318, "Max latency (ns)": 69956547, "Mean latency (ns)": 3413612, "Min duration satisfied": "Yes", "Min latency (ns)": 2516921, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 292.48, "QPS w/o loadgen overhead": 292.94, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 50.242, "characteristics.good": 25121, "characteristics.power": 20.15960333333335, "characteristics.power.normalized_per_core": 2.519950416666669, "characteristics.power.normalized_per_processor": 20.15960333333335, "characteristics.samples_per_second": 292.94, "characteristics.samples_per_second.normalized_per_core": 36.6175, "characteristics.samples_per_second.normalized_per_processor": 292.94, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "e04349b28d212eb0", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3167284, "90.00 percentile latency (ns)": 4934762, "90th percentile latency (ns)": 4934762, "95.00 percentile latency (ns)": 5907637, "97.00 percentile latency (ns)": 6816540, "99.00 percentile latency (ns)": 9020998, "99.90 percentile latency (ns)": 23110114, "Max latency (ns)": 72862640, "Mean latency (ns)": 3701150, "Min duration satisfied": "Yes", "Min latency (ns)": 2675581, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 269.23, "QPS w/o loadgen overhead": 270.19, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 60.842, "characteristics.good": 30421, "characteristics.power": 24.057851666666657, "characteristics.power.normalized_per_core": 3.007231458333332, "characteristics.power.normalized_per_processor": 24.057851666666657, "characteristics.samples_per_second": 270.19, "characteristics.samples_per_second.normalized_per_core": 33.77375, "characteristics.samples_per_second.normalized_per_processor": 270.19, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "8f0b44f5b2743d6b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6575562, "90.00 percentile latency (ns)": 9103972, "90th percentile latency (ns)": 9103972, "95.00 percentile latency (ns)": 11016020, "97.00 percentile latency (ns)": 12343545, "99.00 percentile latency (ns)": 15188400, "99.90 percentile latency (ns)": 35668012, "Max latency (ns)": 128413159, "Mean latency (ns)": 7311525, "Min duration satisfied": "Yes", "Min latency (ns)": 5146733, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 136.46, "QPS w/o loadgen overhead": 136.77, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.016, "characteristics.good": 34508, "characteristics.power": 25.191916666666668, "characteristics.power.normalized_per_core": 3.1489895833333335, "characteristics.power.normalized_per_processor": 25.191916666666668, "characteristics.samples_per_second": 136.77, "characteristics.samples_per_second.normalized_per_core": 17.09625, "characteristics.samples_per_second.normalized_per_processor": 136.77, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "c9911954c0ea339d", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 22649810, "90.00 percentile latency (ns)": 30262782, "90th percentile latency (ns)": 30262782, "95.00 percentile latency (ns)": 32513672, "97.00 percentile latency (ns)": 34583235, "99.00 percentile latency (ns)": 40767709, "99.90 percentile latency (ns)": 78885616, "Max latency (ns)": 157675748, "Mean latency (ns)": 24169196, "Min duration satisfied": "Yes", "Min latency (ns)": 19463689, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 41.35, "QPS w/o loadgen overhead": 41.37, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.098, "characteristics.good": 37049, "characteristics.power": 20.703355, "characteristics.power.normalized_per_core": 2.587919375, "characteristics.power.normalized_per_processor": 20.703355, "characteristics.samples_per_second": 41.37, "characteristics.samples_per_second.normalized_per_core": 5.17125, "characteristics.samples_per_second.normalized_per_processor": 41.37, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite1-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 37.037, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "fa46bb56f5e4d359", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5512629, "90.00 percentile latency (ns)": 8097241, "90th percentile latency (ns)": 8097241, "95.00 percentile latency (ns)": 10121206, "97.00 percentile latency (ns)": 11417179, "99.00 percentile latency (ns)": 13469131, "99.90 percentile latency (ns)": 29671679, "Max latency (ns)": 122685286, "Mean latency (ns)": 6320826, "Min duration satisfied": "Yes", "Min latency (ns)": 4756739, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 157.98, "QPS w/o loadgen overhead": 158.21, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.924, "characteristics.good": 30962, "characteristics.power": 20.18486166666664, "characteristics.power.normalized_per_core": 2.52310770833333, "characteristics.power.normalized_per_processor": 20.18486166666664, "characteristics.samples_per_second": 158.21, "characteristics.samples_per_second.normalized_per_core": 19.77625, "characteristics.samples_per_second.normalized_per_processor": 158.21, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "cef847d49d684616", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5675430, "90.00 percentile latency (ns)": 8679440, "90th percentile latency (ns)": 8679440, "95.00 percentile latency (ns)": 10599911, "97.00 percentile latency (ns)": 11677466, "99.00 percentile latency (ns)": 13510318, "99.90 percentile latency (ns)": 28355290, "Max latency (ns)": 75798944, "Mean latency (ns)": 6434368, "Min duration satisfied": "Yes", "Min latency (ns)": 4948740, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 155.2, "QPS w/o loadgen overhead": 155.42, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 62.298, "characteristics.good": 31149, "characteristics.power": 20.39778166666667, "characteristics.power.normalized_per_core": 2.549722708333334, "characteristics.power.normalized_per_processor": 20.39778166666667, "characteristics.samples_per_second": 155.42, "characteristics.samples_per_second.normalized_per_core": 19.4275, "characteristics.samples_per_second.normalized_per_processor": 155.42, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "aac489bab8eeb9d3", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2754562, "90.00 percentile latency (ns)": 3943099, "90th percentile latency (ns)": 3943099, "95.00 percentile latency (ns)": 4446674, "97.00 percentile latency (ns)": 4824392, "99.00 percentile latency (ns)": 7038218, "99.90 percentile latency (ns)": 21686764, "Max latency (ns)": 129179319, "Mean latency (ns)": 3095085, "Min duration satisfied": "Yes", "Min latency (ns)": 2381040, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 322.52, "QPS w/o loadgen overhead": 323.09, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.12, "characteristics.good": 24560, "characteristics.power": 19.746396666666676, "characteristics.power.normalized_per_core": 2.4682995833333345, "characteristics.power.normalized_per_processor": 19.746396666666676, "characteristics.samples_per_second": 323.09, "characteristics.samples_per_second.normalized_per_core": 40.38625, "characteristics.samples_per_second.normalized_per_processor": 323.09, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "9ae2439994cf0de8", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 22499902, "90.00 percentile latency (ns)": 29464475, "90th percentile latency (ns)": 29464475, "95.00 percentile latency (ns)": 32793618, "97.00 percentile latency (ns)": 36308786, "99.00 percentile latency (ns)": 42190941, "99.90 percentile latency (ns)": 71603351, "Max latency (ns)": 149480489, "Mean latency (ns)": 23939449, "Min duration satisfied": "Yes", "Min latency (ns)": 19285101, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 41.75, "QPS w/o loadgen overhead": 41.77, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.584, "characteristics.good": 36792, "characteristics.power": 20.620281666666646, "characteristics.power.normalized_per_core": 2.5775352083333307, "characteristics.power.normalized_per_processor": 20.620281666666646, "characteristics.samples_per_second": 41.77, "characteristics.samples_per_second.normalized_per_core": 5.22125, "characteristics.samples_per_second.normalized_per_processor": 41.77, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.4-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.4-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_140.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "9d7a4fbabdd484c9", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 16301685, "90.00 percentile latency (ns)": 19529412, "90th percentile latency (ns)": 19529412, "95.00 percentile latency (ns)": 21500153, "97.00 percentile latency (ns)": 22566635, "99.00 percentile latency (ns)": 26088867, "99.90 percentile latency (ns)": 52406256, "Max latency (ns)": 93765880, "Mean latency (ns)": 16976853, "Min duration satisfied": "Yes", "Min latency (ns)": 14077841, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 58.86, "QPS w/o loadgen overhead": 58.9, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.772, "characteristics.good": 35386, "characteristics.power": 20.585893333333335, "characteristics.power.normalized_per_core": 2.573236666666667, "characteristics.power.normalized_per_processor": 20.585893333333335, "characteristics.samples_per_second": 58.9, "characteristics.samples_per_second.normalized_per_core": 7.3625, "characteristics.samples_per_second.normalized_per_processor": 58.9, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 55.5556, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "e4db4c59aff951ce", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 1868215, "90.00 percentile latency (ns)": 2594904, "90th percentile latency (ns)": 2594904, "95.00 percentile latency (ns)": 2988938, "97.00 percentile latency (ns)": 3418879, "99.00 percentile latency (ns)": 4512690, "99.90 percentile latency (ns)": 16145899, "Max latency (ns)": 68488992, "Mean latency (ns)": 2072893, "Min duration satisfied": "Yes", "Min latency (ns)": 1608586, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 481.03, "QPS w/o loadgen overhead": 482.42, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 47.808, "characteristics.good": 23904, "characteristics.power": 22.398023333333366, "characteristics.power.normalized_per_core": 2.799752916666671, "characteristics.power.normalized_per_processor": 22.398023333333366, "characteristics.samples_per_second": 482.42, "characteristics.samples_per_second.normalized_per_core": 60.3025, "characteristics.samples_per_second.normalized_per_processor": 482.42, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "aeb479b42f39fabf", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3682414, "90.00 percentile latency (ns)": 5579233, "90th percentile latency (ns)": 5579233, "95.00 percentile latency (ns)": 6600587, "97.00 percentile latency (ns)": 7782183, "99.00 percentile latency (ns)": 10658674, "99.90 percentile latency (ns)": 27272082, "Max latency (ns)": 123568126, "Mean latency (ns)": 4277698, "Min duration satisfied": "Yes", "Min latency (ns)": 2938248, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 232.94, "QPS w/o loadgen overhead": 233.77, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.038, "characteristics.good": 32019, "characteristics.power": 23.769883333333333, "characteristics.power.normalized_per_core": 2.9712354166666666, "characteristics.power.normalized_per_processor": 23.769883333333333, "characteristics.samples_per_second": 233.77, "characteristics.samples_per_second.normalized_per_core": 29.22125, "characteristics.samples_per_second.normalized_per_processor": 233.77, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "1bf0f4353889a29c", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3149970, "90.00 percentile latency (ns)": 4029306, "90th percentile latency (ns)": 4029306, "95.00 percentile latency (ns)": 4722618, "97.00 percentile latency (ns)": 5274996, "99.00 percentile latency (ns)": 7352887, "99.90 percentile latency (ns)": 21464031, "Max latency (ns)": 82111275, "Mean latency (ns)": 3435341, "Min duration satisfied": "Yes", "Min latency (ns)": 2695420, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 290.22, "QPS w/o loadgen overhead": 291.09, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 51.848, "characteristics.good": 25924, "characteristics.power": 23.46950000000002, "characteristics.power.normalized_per_core": 2.9336875000000027, "characteristics.power.normalized_per_processor": 23.46950000000002, "characteristics.samples_per_second": 291.09, "characteristics.samples_per_second.normalized_per_core": 36.38625, "characteristics.samples_per_second.normalized_per_processor": 291.09, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "08ae52214ff08a07", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4655066, "90.00 percentile latency (ns)": 6596660, "90th percentile latency (ns)": 6596660, "95.00 percentile latency (ns)": 8217536, "97.00 percentile latency (ns)": 9571745, "99.00 percentile latency (ns)": 11924108, "99.90 percentile latency (ns)": 26088262, "Max latency (ns)": 125972920, "Mean latency (ns)": 5287509, "Min duration satisfied": "Yes", "Min latency (ns)": 3957465, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 188.84, "QPS w/o loadgen overhead": 189.12, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.226, "characteristics.good": 29613, "characteristics.power": 19.70921833333334, "characteristics.power.normalized_per_core": 2.4636522916666674, "characteristics.power.normalized_per_processor": 19.70921833333334, "characteristics.samples_per_second": 189.12, "characteristics.samples_per_second.normalized_per_core": 23.64, "characteristics.samples_per_second.normalized_per_processor": 189.12, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "078235a45fb9443a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9626867, "90.00 percentile latency (ns)": 11938267, "90th percentile latency (ns)": 11938267, "95.00 percentile latency (ns)": 14228675, "97.00 percentile latency (ns)": 15428633, "99.00 percentile latency (ns)": 18222744, "99.90 percentile latency (ns)": 39503609, "Max latency (ns)": 79216507, "Mean latency (ns)": 10337667, "Min duration satisfied": "Yes", "Min latency (ns)": 8190387, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 96.62, "QPS w/o loadgen overhead": 96.73, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.248, "characteristics.good": 34124, "characteristics.power": 20.464108333333357, "characteristics.power.normalized_per_core": 2.5580135416666696, "characteristics.power.normalized_per_processor": 20.464108333333357, "characteristics.samples_per_second": 96.73, "characteristics.samples_per_second.normalized_per_core": 12.09125, "characteristics.samples_per_second.normalized_per_processor": 96.73, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "f7412f27a42918b3", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4378086, "90.00 percentile latency (ns)": 6804404, "90th percentile latency (ns)": 6804404, "95.00 percentile latency (ns)": 8571427, "97.00 percentile latency (ns)": 9954785, "99.00 percentile latency (ns)": 12294122, "99.90 percentile latency (ns)": 27509494, "Max latency (ns)": 128000686, "Mean latency (ns)": 5142050, "Min duration satisfied": "Yes", "Min latency (ns)": 3727176, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 194.21, "QPS w/o loadgen overhead": 194.47, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.12, "characteristics.good": 29560, "characteristics.power": 19.780171666666657, "characteristics.power.normalized_per_core": 2.472521458333332, "characteristics.power.normalized_per_processor": 19.780171666666657, "characteristics.samples_per_second": 194.47, "characteristics.samples_per_second.normalized_per_core": 24.30875, "characteristics.samples_per_second.normalized_per_processor": 194.47, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "e63e46dabf2c9622", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4523192, "90.00 percentile latency (ns)": 6214310, "90th percentile latency (ns)": 6214310, "95.00 percentile latency (ns)": 6917674, "97.00 percentile latency (ns)": 8119558, "99.00 percentile latency (ns)": 11018002, "99.90 percentile latency (ns)": 24635037, "Max latency (ns)": 71195600, "Mean latency (ns)": 5058455, "Min duration satisfied": "Yes", "Min latency (ns)": 4011043, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 197.4, "QPS w/o loadgen overhead": 197.69, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.46, "characteristics.good": 29230, "characteristics.power": 20.143808333333336, "characteristics.power.normalized_per_core": 2.517976041666667, "characteristics.power.normalized_per_processor": 20.143808333333336, "characteristics.samples_per_second": 197.69, "characteristics.samples_per_second.normalized_per_core": 24.71125, "characteristics.samples_per_second.normalized_per_processor": 197.69, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "8cd4288bcba6b7b5", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 12917624, "90.00 percentile latency (ns)": 18922679, "90th percentile latency (ns)": 18922679, "95.00 percentile latency (ns)": 20467190, "97.00 percentile latency (ns)": 21640216, "99.00 percentile latency (ns)": 25918506, "99.90 percentile latency (ns)": 46384423, "Max latency (ns)": 128637499, "Mean latency (ns)": 14136954, "Min duration satisfied": "Yes", "Min latency (ns)": 10903074, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 70.66, "QPS w/o loadgen overhead": 70.74, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.91, "characteristics.good": 36455, "characteristics.power": 20.698503333333345, "characteristics.power.normalized_per_core": 2.587312916666668, "characteristics.power.normalized_per_processor": 20.698503333333345, "characteristics.samples_per_second": 70.74, "characteristics.samples_per_second.normalized_per_core": 8.8425, "characteristics.samples_per_second.normalized_per_processor": 70.74, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-large_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 62.5, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "c3698e2585423c1f", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 14173416, "90.00 percentile latency (ns)": 21286241, "90th percentile latency (ns)": 21286241, "95.00 percentile latency (ns)": 24615131, "97.00 percentile latency (ns)": 27119050, "99.00 percentile latency (ns)": 30658809, "99.90 percentile latency (ns)": 58881312, "Max latency (ns)": 137478978, "Mean latency (ns)": 15724905, "Min duration satisfied": "Yes", "Min latency (ns)": 11302529, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 63.54, "QPS w/o loadgen overhead": 63.59, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.542, "characteristics.good": 35271, "characteristics.power": 19.97022833333332, "characteristics.power.normalized_per_core": 2.496278541666665, "characteristics.power.normalized_per_processor": 19.97022833333332, "characteristics.samples_per_second": 63.59, "characteristics.samples_per_second.normalized_per_core": 7.94875, "characteristics.samples_per_second.normalized_per_processor": 63.59, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 55.5556, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "d59aa87ad72ff55d", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4591710, "90.00 percentile latency (ns)": 6385803, "90th percentile latency (ns)": 6385803, "95.00 percentile latency (ns)": 7656678, "97.00 percentile latency (ns)": 9014886, "99.00 percentile latency (ns)": 11705945, "99.90 percentile latency (ns)": 26008317, "Max latency (ns)": 112191236, "Mean latency (ns)": 5177181, "Min duration satisfied": "Yes", "Min latency (ns)": 3957242, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 192.89, "QPS w/o loadgen overhead": 193.16, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.594, "characteristics.good": 28297, "characteristics.power": 19.982586666666645, "characteristics.power.normalized_per_core": 2.4978233333333306, "characteristics.power.normalized_per_processor": 19.982586666666645, "characteristics.samples_per_second": 193.16, "characteristics.samples_per_second.normalized_per_core": 24.145, "characteristics.samples_per_second.normalized_per_processor": 193.16, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "4617b89549c35e46", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8176375, "90.00 percentile latency (ns)": 13853309, "90th percentile latency (ns)": 13853309, "95.00 percentile latency (ns)": 15363650, "97.00 percentile latency (ns)": 16305037, "99.00 percentile latency (ns)": 19024073, "99.90 percentile latency (ns)": 38040819, "Max latency (ns)": 132926394, "Mean latency (ns)": 9393196, "Min duration satisfied": "Yes", "Min latency (ns)": 6567821, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 106.34, "QPS w/o loadgen overhead": 106.46, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.82, "characteristics.good": 33910, "characteristics.power": 20.44295333333332, "characteristics.power.normalized_per_core": 2.555369166666665, "characteristics.power.normalized_per_processor": 20.44295333333332, "characteristics.samples_per_second": 106.46, "characteristics.samples_per_second.normalized_per_core": 13.3075, "characteristics.samples_per_second.normalized_per_processor": 106.46, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "b46f68de2dbf1db2", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 1558507, "90.00 percentile latency (ns)": 2268546, "90th percentile latency (ns)": 2268546, "95.00 percentile latency (ns)": 2682223, "97.00 percentile latency (ns)": 3056785, "99.00 percentile latency (ns)": 3914927, "99.90 percentile latency (ns)": 13227837, "Max latency (ns)": 73961783, "Mean latency (ns)": 1753878, "Min duration satisfied": "Yes", "Min latency (ns)": 1331872, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 568.65, "QPS w/o loadgen overhead": 570.17, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 44.116, "characteristics.good": 22058, "characteristics.power": 21.867051666666665, "characteristics.power.normalized_per_core": 2.733381458333333, "characteristics.power.normalized_per_processor": 21.867051666666665, "characteristics.samples_per_second": 570.17, "characteristics.samples_per_second.normalized_per_core": 71.27125, "characteristics.samples_per_second.normalized_per_processor": 570.17, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "37e6541c0986f7c6", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2068094, "90.00 percentile latency (ns)": 3030412, "90th percentile latency (ns)": 3030412, "95.00 percentile latency (ns)": 3568197, "97.00 percentile latency (ns)": 3976951, "99.00 percentile latency (ns)": 5664389, "99.90 percentile latency (ns)": 15547597, "Max latency (ns)": 62200196, "Mean latency (ns)": 2323470, "Min duration satisfied": "Yes", "Min latency (ns)": 1710702, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 429.59, "QPS w/o loadgen overhead": 430.39, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 45.494, "characteristics.good": 22747, "characteristics.power": 20.08869833333332, "characteristics.power.normalized_per_core": 2.511087291666665, "characteristics.power.normalized_per_processor": 20.08869833333332, "characteristics.samples_per_second": 430.39, "characteristics.samples_per_second.normalized_per_core": 53.79875, "characteristics.samples_per_second.normalized_per_processor": 430.39, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "2244fa85ceebbbf5", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11473480, "90.00 percentile latency (ns)": 15845086, "90th percentile latency (ns)": 15845086, "95.00 percentile latency (ns)": 17058437, "97.00 percentile latency (ns)": 17949998, "99.00 percentile latency (ns)": 22140171, "99.90 percentile latency (ns)": 41552937, "Max latency (ns)": 82658104, "Mean latency (ns)": 12462055, "Min duration satisfied": "Yes", "Min latency (ns)": 8534083, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 80.12, "QPS w/o loadgen overhead": 80.24, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.626, "characteristics.good": 35313, "characteristics.power": 26.28694999999998, "characteristics.power.normalized_per_core": 3.2858687499999975, "characteristics.power.normalized_per_processor": 26.28694999999998, "characteristics.samples_per_second": 80.24, "characteristics.samples_per_second.normalized_per_core": 10.03, "characteristics.samples_per_second.normalized_per_processor": 80.24, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "58b04b0fab4fd89e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6354116, "90.00 percentile latency (ns)": 9581751, "90th percentile latency (ns)": 9581751, "95.00 percentile latency (ns)": 11643172, "97.00 percentile latency (ns)": 12851213, "99.00 percentile latency (ns)": 14993594, "99.90 percentile latency (ns)": 33768461, "Max latency (ns)": 128921015, "Mean latency (ns)": 7224611, "Min duration satisfied": "Yes", "Min latency (ns)": 5120138, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 138.24, "QPS w/o loadgen overhead": 138.42, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.454, "characteristics.good": 31727, "characteristics.power": 20.199669999999994, "characteristics.power.normalized_per_core": 2.5249587499999993, "characteristics.power.normalized_per_processor": 20.199669999999994, "characteristics.samples_per_second": 138.42, "characteristics.samples_per_second.normalized_per_core": 17.3025, "characteristics.samples_per_second.normalized_per_processor": 138.42, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "09c742e6945ce24f", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 12063631, "90.00 percentile latency (ns)": 14007817, "90th percentile latency (ns)": 14007817, "95.00 percentile latency (ns)": 15204704, "97.00 percentile latency (ns)": 16719939, "99.00 percentile latency (ns)": 20739164, "99.90 percentile latency (ns)": 46602609, "Max latency (ns)": 83461607, "Mean latency (ns)": 12639288, "Min duration satisfied": "Yes", "Min latency (ns)": 10452994, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 79.03, "QPS w/o loadgen overhead": 79.12, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.04, "characteristics.good": 35020, "characteristics.power": 20.656341666666645, "characteristics.power.normalized_per_core": 2.5820427083333306, "characteristics.power.normalized_per_processor": 20.656341666666645, "characteristics.samples_per_second": 79.12, "characteristics.samples_per_second.normalized_per_core": 9.89, "characteristics.samples_per_second.normalized_per_processor": 79.12, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "ae703e690d64a9ea", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 30677651, "90.00 percentile latency (ns)": 41117917, "90th percentile latency (ns)": 41117917, "95.00 percentile latency (ns)": 44795886, "97.00 percentile latency (ns)": 48510785, "99.00 percentile latency (ns)": 54832846, "99.90 percentile latency (ns)": 96377529, "Max latency (ns)": 154675726, "Mean latency (ns)": 32949910, "Min duration satisfied": "Yes", "Min latency (ns)": 26110754, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 30.33, "QPS w/o loadgen overhead": 30.35, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.124, "characteristics.good": 37062, "characteristics.power": 21.019524999999987, "characteristics.power.normalized_per_core": 2.6274406249999984, "characteristics.power.normalized_per_processor": 21.019524999999987, "characteristics.samples_per_second": 30.35, "characteristics.samples_per_second.normalized_per_core": 3.79375, "characteristics.samples_per_second.normalized_per_processor": 30.35, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite2-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 27.7778, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "9f7cec6e47859672", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3418906, "90.00 percentile latency (ns)": 5402612, "90th percentile latency (ns)": 5402612, "95.00 percentile latency (ns)": 6862680, "97.00 percentile latency (ns)": 8040460, "99.00 percentile latency (ns)": 10257266, "99.90 percentile latency (ns)": 23474084, "Max latency (ns)": 76043495, "Mean latency (ns)": 4038549, "Min duration satisfied": "Yes", "Min latency (ns)": 2937861, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 247.23, "QPS w/o loadgen overhead": 247.61, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.444, "characteristics.good": 29722, "characteristics.power": 20.116530000000008, "characteristics.power.normalized_per_core": 2.514566250000001, "characteristics.power.normalized_per_processor": 20.116530000000008, "characteristics.samples_per_second": 247.61, "characteristics.samples_per_second.normalized_per_core": 30.95125, "characteristics.samples_per_second.normalized_per_processor": 247.61, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "a4c778c9762e44ad", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4461975, "90.00 percentile latency (ns)": 6648225, "90th percentile latency (ns)": 6648225, "95.00 percentile latency (ns)": 8682659, "97.00 percentile latency (ns)": 9794808, "99.00 percentile latency (ns)": 11635537, "99.90 percentile latency (ns)": 26625187, "Max latency (ns)": 72033836, "Mean latency (ns)": 5147098, "Min duration satisfied": "Yes", "Min latency (ns)": 3961599, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 194.01, "QPS w/o loadgen overhead": 194.28, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.456, "characteristics.good": 30728, "characteristics.power": 20.206338333333335, "characteristics.power.normalized_per_core": 2.525792291666667, "characteristics.power.normalized_per_processor": 20.206338333333335, "characteristics.samples_per_second": 194.28, "characteristics.samples_per_second.normalized_per_core": 24.285, "characteristics.samples_per_second.normalized_per_processor": 194.28, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "d4f2ec8464e87866", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5823134, "90.00 percentile latency (ns)": 7851284, "90th percentile latency (ns)": 7851284, "95.00 percentile latency (ns)": 9964376, "97.00 percentile latency (ns)": 11345012, "99.00 percentile latency (ns)": 13578001, "99.90 percentile latency (ns)": 29961386, "Max latency (ns)": 120233923, "Mean latency (ns)": 6477502, "Min duration satisfied": "Yes", "Min latency (ns)": 4939795, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 154.18, "QPS w/o loadgen overhead": 154.38, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.528, "characteristics.good": 29264, "characteristics.power": 19.84498500000002, "characteristics.power.normalized_per_core": 2.4806231250000024, "characteristics.power.normalized_per_processor": 19.84498500000002, "characteristics.samples_per_second": 154.38, "characteristics.samples_per_second.normalized_per_core": 19.2975, "characteristics.samples_per_second.normalized_per_processor": 154.38, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "fb7acaa1244f8ee8", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4504046, "90.00 percentile latency (ns)": 6418373, "90th percentile latency (ns)": 6418373, "95.00 percentile latency (ns)": 7603770, "97.00 percentile latency (ns)": 8731055, "99.00 percentile latency (ns)": 11597106, "99.90 percentile latency (ns)": 28114500, "Max latency (ns)": 120350900, "Mean latency (ns)": 5070490, "Min duration satisfied": "Yes", "Min latency (ns)": 3686921, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 196.65, "QPS w/o loadgen overhead": 197.22, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.036, "characteristics.good": 33018, "characteristics.power": 24.566499999999976, "characteristics.power.normalized_per_core": 3.070812499999997, "characteristics.power.normalized_per_processor": 24.566499999999976, "characteristics.samples_per_second": 197.22, "characteristics.samples_per_second.normalized_per_core": 24.6525, "characteristics.samples_per_second.normalized_per_processor": 197.22, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "124d148406c8dc3b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 19622143, "90.00 percentile latency (ns)": 26600140, "90th percentile latency (ns)": 26600140, "95.00 percentile latency (ns)": 28258624, "97.00 percentile latency (ns)": 29684798, "99.00 percentile latency (ns)": 35725489, "99.90 percentile latency (ns)": 62252058, "Max latency (ns)": 166883444, "Mean latency (ns)": 21095843, "Min duration satisfied": "Yes", "Min latency (ns)": 14512555, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 47.36, "QPS w/o loadgen overhead": 47.4, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.642, "characteristics.good": 38321, "characteristics.power": 26.411166666666695, "characteristics.power.normalized_per_core": 3.301395833333337, "characteristics.power.normalized_per_processor": 26.411166666666695, "characteristics.samples_per_second": 47.4, "characteristics.samples_per_second.normalized_per_core": 5.925, "characteristics.samples_per_second.normalized_per_processor": 47.4, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite1-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "c44d29f21129cfa6", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10808280, "90.00 percentile latency (ns)": 17635924, "90th percentile latency (ns)": 17635924, "95.00 percentile latency (ns)": 19864076, "97.00 percentile latency (ns)": 21164633, "99.00 percentile latency (ns)": 23886408, "99.90 percentile latency (ns)": 43315983, "Max latency (ns)": 147077335, "Mean latency (ns)": 12208508, "Min duration satisfied": "Yes", "Min latency (ns)": 8656759, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 81.83, "QPS w/o loadgen overhead": 81.91, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.604, "characteristics.good": 34802, "characteristics.power": 20.33190000000001, "characteristics.power.normalized_per_core": 2.5414875000000015, "characteristics.power.normalized_per_processor": 20.33190000000001, "characteristics.samples_per_second": 81.91, "characteristics.samples_per_second.normalized_per_core": 10.23875, "characteristics.samples_per_second.normalized_per_processor": 81.91, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "ce1265e2c42bb8db", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8903933, "90.00 percentile latency (ns)": 12755434, "90th percentile latency (ns)": 12755434, "95.00 percentile latency (ns)": 14305979, "97.00 percentile latency (ns)": 15233455, "99.00 percentile latency (ns)": 18666456, "99.90 percentile latency (ns)": 39204426, "Max latency (ns)": 82310036, "Mean latency (ns)": 9814902, "Min duration satisfied": "Yes", "Min latency (ns)": 7681346, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 101.69, "QPS w/o loadgen overhead": 101.89, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.262, "characteristics.good": 34631, "characteristics.power": 25.68110000000002, "characteristics.power.normalized_per_core": 3.2101375000000023, "characteristics.power.normalized_per_processor": 25.68110000000002, "characteristics.samples_per_second": 101.89, "characteristics.samples_per_second.normalized_per_core": 12.73625, "characteristics.samples_per_second.normalized_per_processor": 101.89, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "e33b48cc903ff00e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11598285, "90.00 percentile latency (ns)": 16323818, "90th percentile latency (ns)": 16323818, "95.00 percentile latency (ns)": 18504613, "97.00 percentile latency (ns)": 20045318, "99.00 percentile latency (ns)": 23341807, "99.90 percentile latency (ns)": 42834237, "Max latency (ns)": 123304072, "Mean latency (ns)": 12565286, "Min duration satisfied": "Yes", "Min latency (ns)": 9537039, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 79.51, "QPS w/o loadgen overhead": 79.58, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.734, "characteristics.good": 34367, "characteristics.power": 20.209103333333324, "characteristics.power.normalized_per_core": 2.5261379166666655, "characteristics.power.normalized_per_processor": 20.209103333333324, "characteristics.samples_per_second": 79.58, "characteristics.samples_per_second.normalized_per_core": 9.9475, "characteristics.samples_per_second.normalized_per_processor": 79.58, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 62.5, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "c46258d6f065e14f", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3943108, "90.00 percentile latency (ns)": 5910070, "90th percentile latency (ns)": 5910070, "95.00 percentile latency (ns)": 6983490, "97.00 percentile latency (ns)": 7965195, "99.00 percentile latency (ns)": 10565495, "99.90 percentile latency (ns)": 26224802, "Max latency (ns)": 88853610, "Mean latency (ns)": 4519135, "Min duration satisfied": "Yes", "Min latency (ns)": 3322410, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 220.58, "QPS w/o loadgen overhead": 221.28, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.33, "characteristics.good": 31665, "characteristics.power": 22.367974999999984, "characteristics.power.normalized_per_core": 2.795996874999998, "characteristics.power.normalized_per_processor": 22.367974999999984, "characteristics.samples_per_second": 221.28, "characteristics.samples_per_second.normalized_per_core": 27.66, "characteristics.samples_per_second.normalized_per_processor": 221.28, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "d31aa2449cfb1194", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2193286, "90.00 percentile latency (ns)": 3544742, "90th percentile latency (ns)": 3544742, "95.00 percentile latency (ns)": 4214948, "97.00 percentile latency (ns)": 4773888, "99.00 percentile latency (ns)": 7036779, "99.90 percentile latency (ns)": 21136819, "Max latency (ns)": 81606746, "Mean latency (ns)": 2576407, "Min duration satisfied": "Yes", "Min latency (ns)": 1842614, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 386.65, "QPS w/o loadgen overhead": 388.14, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 57.918, "characteristics.good": 28959, "characteristics.power": 23.64076666666669, "characteristics.power.normalized_per_core": 2.955095833333336, "characteristics.power.normalized_per_processor": 23.64076666666669, "characteristics.samples_per_second": 388.14, "characteristics.samples_per_second.normalized_per_core": 48.5175, "characteristics.samples_per_second.normalized_per_processor": 388.14, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "3f82ee56728628d6", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 14315235, "90.00 percentile latency (ns)": 19040756, "90th percentile latency (ns)": 19040756, "95.00 percentile latency (ns)": 20308550, "97.00 percentile latency (ns)": 21306300, "99.00 percentile latency (ns)": 26206308, "99.90 percentile latency (ns)": 51444309, "Max latency (ns)": 96473327, "Mean latency (ns)": 15175138, "Min duration satisfied": "Yes", "Min latency (ns)": 11242178, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 65.81, "QPS w/o loadgen overhead": 65.9, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.676, "characteristics.good": 35838, "characteristics.power": 26.43891666666667, "characteristics.power.normalized_per_core": 3.304864583333334, "characteristics.power.normalized_per_processor": 26.43891666666667, "characteristics.samples_per_second": 65.9, "characteristics.samples_per_second.normalized_per_core": 8.2375, "characteristics.samples_per_second.normalized_per_processor": 65.9, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "6b246f584f6ba1ce", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 1988325, "90.00 percentile latency (ns)": 2998265, "90th percentile latency (ns)": 2998265, "95.00 percentile latency (ns)": 3547392, "97.00 percentile latency (ns)": 3909136, "99.00 percentile latency (ns)": 5479634, "99.90 percentile latency (ns)": 16326948, "Max latency (ns)": 97895517, "Mean latency (ns)": 2260297, "Min duration satisfied": "Yes", "Min latency (ns)": 1647509, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 441.6, "QPS w/o loadgen overhead": 442.42, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 43.468, "characteristics.good": 21734, "characteristics.power": 19.81896666666671, "characteristics.power.normalized_per_core": 2.477370833333339, "characteristics.power.normalized_per_processor": 19.81896666666671, "characteristics.samples_per_second": 442.42, "characteristics.samples_per_second.normalized_per_core": 55.3025, "characteristics.samples_per_second.normalized_per_processor": 442.42, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "6b78549c84d63825", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 63751339, "90.00 percentile latency (ns)": 76286615, "90th percentile latency (ns)": 76286615, "95.00 percentile latency (ns)": 80711803, "97.00 percentile latency (ns)": 83761760, "99.00 percentile latency (ns)": 91016661, "99.90 percentile latency (ns)": 171047730, "Max latency (ns)": 201351097, "Mean latency (ns)": 64052072, "Min duration satisfied": "Yes", "Min latency (ns)": 48646093, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 15.61, "QPS w/o loadgen overhead": 15.61, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 80.676, "characteristics.good": 40338, "characteristics.power": 27.748951747088228, "characteristics.power.normalized_per_core": 3.4686189683860285, "characteristics.power.normalized_per_processor": 27.748951747088228, "characteristics.samples_per_second": 15.61, "characteristics.samples_per_second.normalized_per_core": 1.95125, "characteristics.samples_per_second.normalized_per_processor": 15.61, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite4-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.625, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "ddaa85939d7d4b0a", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10325941, "90.00 percentile latency (ns)": 13457448, "90th percentile latency (ns)": 13457448, "95.00 percentile latency (ns)": 16154810, "97.00 percentile latency (ns)": 17241113, "99.00 percentile latency (ns)": 20476250, "99.90 percentile latency (ns)": 39783919, "Max latency (ns)": 139200805, "Mean latency (ns)": 11167141, "Min duration satisfied": "Yes", "Min latency (ns)": 8860169, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 89.44, "QPS w/o loadgen overhead": 89.55, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.43, "characteristics.good": 35215, "characteristics.power": 20.373743333333334, "characteristics.power.normalized_per_core": 2.5467179166666667, "characteristics.power.normalized_per_processor": 20.373743333333334, "characteristics.samples_per_second": 89.55, "characteristics.samples_per_second.normalized_per_core": 11.19375, "characteristics.samples_per_second.normalized_per_processor": 89.55, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "a9747af950306ad4", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8810711, "90.00 percentile latency (ns)": 13262278, "90th percentile latency (ns)": 13262278, "95.00 percentile latency (ns)": 15494316, "97.00 percentile latency (ns)": 16504859, "99.00 percentile latency (ns)": 19357341, "99.90 percentile latency (ns)": 39296597, "Max latency (ns)": 136397492, "Mean latency (ns)": 9849464, "Min duration satisfied": "Yes", "Min latency (ns)": 7473433, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 101.42, "QPS w/o loadgen overhead": 101.53, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.474, "characteristics.good": 33737, "characteristics.power": 20.269488333333346, "characteristics.power.normalized_per_core": 2.533686041666668, "characteristics.power.normalized_per_processor": 20.269488333333346, "characteristics.samples_per_second": 101.53, "characteristics.samples_per_second.normalized_per_core": 12.69125, "characteristics.samples_per_second.normalized_per_processor": 101.53, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "1e3d91e6c4c90aec", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8446451, "90.00 percentile latency (ns)": 10151434, "90th percentile latency (ns)": 10151434, "95.00 percentile latency (ns)": 10944666, "97.00 percentile latency (ns)": 12585929, "99.00 percentile latency (ns)": 16119733, "99.90 percentile latency (ns)": 32293832, "Max latency (ns)": 81263623, "Mean latency (ns)": 8952275, "Min duration satisfied": "Yes", "Min latency (ns)": 7197140, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 111.58, "QPS w/o loadgen overhead": 111.7, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.24, "characteristics.good": 33620, "characteristics.power": 20.42497666666668, "characteristics.power.normalized_per_core": 2.553122083333335, "characteristics.power.normalized_per_processor": 20.42497666666668, "characteristics.samples_per_second": 111.7, "characteristics.samples_per_second.normalized_per_core": 13.9625, "characteristics.samples_per_second.normalized_per_processor": 111.7, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "8c7a8459ba6b2b74", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5540546, "90.00 percentile latency (ns)": 8239409, "90th percentile latency (ns)": 8239409, "95.00 percentile latency (ns)": 9717285, "97.00 percentile latency (ns)": 10817187, "99.00 percentile latency (ns)": 12993417, "99.90 percentile latency (ns)": 30217515, "Max latency (ns)": 92696539, "Mean latency (ns)": 6279251, "Min duration satisfied": "Yes", "Min latency (ns)": 4636504, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 158.87, "QPS w/o loadgen overhead": 159.25, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.348, "characteristics.good": 32174, "characteristics.power": 24.942266666666665, "characteristics.power.normalized_per_core": 3.117783333333333, "characteristics.power.normalized_per_processor": 24.942266666666665, "characteristics.samples_per_second": 159.25, "characteristics.samples_per_second.normalized_per_core": 19.90625, "characteristics.samples_per_second.normalized_per_processor": 159.25, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "154e5c9356d520ae", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4365493, "90.00 percentile latency (ns)": 6893874, "90th percentile latency (ns)": 6893874, "95.00 percentile latency (ns)": 8111629, "97.00 percentile latency (ns)": 9101988, "99.00 percentile latency (ns)": 11314572, "99.90 percentile latency (ns)": 26381125, "Max latency (ns)": 81638393, "Mean latency (ns)": 5068228, "Min duration satisfied": "Yes", "Min latency (ns)": 3580689, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 196.7, "QPS w/o loadgen overhead": 197.31, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.246, "characteristics.good": 31623, "characteristics.power": 24.486800000000017, "characteristics.power.normalized_per_core": 3.060850000000002, "characteristics.power.normalized_per_processor": 24.486800000000017, "characteristics.samples_per_second": 197.31, "characteristics.samples_per_second.normalized_per_core": 24.66375, "characteristics.samples_per_second.normalized_per_processor": 197.31, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "7bb4cccad98e8f94", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3527882, "90.00 percentile latency (ns)": 5098996, "90th percentile latency (ns)": 5098996, "95.00 percentile latency (ns)": 5644430, "97.00 percentile latency (ns)": 6244660, "99.00 percentile latency (ns)": 9220075, "99.90 percentile latency (ns)": 26543955, "Max latency (ns)": 119870983, "Mean latency (ns)": 3966579, "Min duration satisfied": "Yes", "Min latency (ns)": 2913010, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 251.26, "QPS w/o loadgen overhead": 252.11, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.668, "characteristics.good": 30834, "characteristics.power": 23.48276666666667, "characteristics.power.normalized_per_core": 2.9353458333333338, "characteristics.power.normalized_per_processor": 23.48276666666667, "characteristics.samples_per_second": 252.11, "characteristics.samples_per_second.normalized_per_core": 31.51375, "characteristics.samples_per_second.normalized_per_processor": 252.11, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "cb2f0af1a4c52a34", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5850949, "90.00 percentile latency (ns)": 8744358, "90th percentile latency (ns)": 8744358, "95.00 percentile latency (ns)": 10179398, "97.00 percentile latency (ns)": 11212372, "99.00 percentile latency (ns)": 13646881, "99.90 percentile latency (ns)": 31531774, "Max latency (ns)": 79257967, "Mean latency (ns)": 6607980, "Min duration satisfied": "Yes", "Min latency (ns)": 4755924, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 150.97, "QPS w/o loadgen overhead": 151.33, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.384, "characteristics.good": 33192, "characteristics.power": 25.208666666666673, "characteristics.power.normalized_per_core": 3.151083333333334, "characteristics.power.normalized_per_processor": 25.208666666666673, "characteristics.samples_per_second": 151.33, "characteristics.samples_per_second.normalized_per_core": 18.91625, "characteristics.samples_per_second.normalized_per_processor": 151.33, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "5a67d5fcf40df954", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2613460, "90.00 percentile latency (ns)": 3881941, "90th percentile latency (ns)": 3881941, "95.00 percentile latency (ns)": 4463845, "97.00 percentile latency (ns)": 5076224, "99.00 percentile latency (ns)": 7736857, "99.90 percentile latency (ns)": 18208961, "Max latency (ns)": 105302971, "Mean latency (ns)": 2977990, "Min duration satisfied": "Yes", "Min latency (ns)": 2266884, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 335.15, "QPS w/o loadgen overhead": 335.8, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.11, "characteristics.good": 24555, "characteristics.power": 20.14089500000001, "characteristics.power.normalized_per_core": 2.5176118750000014, "characteristics.power.normalized_per_processor": 20.14089500000001, "characteristics.samples_per_second": 335.8, "characteristics.samples_per_second.normalized_per_core": 41.975, "characteristics.samples_per_second.normalized_per_processor": 335.8, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "c8a0573746e595a7", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5092801, "90.00 percentile latency (ns)": 7258691, "90th percentile latency (ns)": 7258691, "95.00 percentile latency (ns)": 9254396, "97.00 percentile latency (ns)": 10704896, "99.00 percentile latency (ns)": 12977216, "99.90 percentile latency (ns)": 28647297, "Max latency (ns)": 119657356, "Mean latency (ns)": 5841650, "Min duration satisfied": "Yes", "Min latency (ns)": 4493898, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 170.93, "QPS w/o loadgen overhead": 171.18, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 62.07, "characteristics.good": 31035, "characteristics.power": 19.565104999999996, "characteristics.power.normalized_per_core": 2.4456381249999994, "characteristics.power.normalized_per_processor": 19.565104999999996, "characteristics.samples_per_second": 171.18, "characteristics.samples_per_second.normalized_per_core": 21.3975, "characteristics.samples_per_second.normalized_per_processor": 171.18, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "927ce9be7481fb7d", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3510516, "90.00 percentile latency (ns)": 5990814, "90th percentile latency (ns)": 5990814, "95.00 percentile latency (ns)": 7273493, "97.00 percentile latency (ns)": 8260874, "99.00 percentile latency (ns)": 10812067, "99.90 percentile latency (ns)": 27799791, "Max latency (ns)": 131605063, "Mean latency (ns)": 4252716, "Min duration satisfied": "Yes", "Min latency (ns)": 2743608, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 234.24, "QPS w/o loadgen overhead": 235.14, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.224, "characteristics.good": 30612, "characteristics.power": 24.067116666666678, "characteristics.power.normalized_per_core": 3.0083895833333347, "characteristics.power.normalized_per_processor": 24.067116666666678, "characteristics.samples_per_second": 235.14, "characteristics.samples_per_second.normalized_per_core": 29.3925, "characteristics.samples_per_second.normalized_per_processor": 235.14, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "96e59b2cdd4b22be", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 1638246, "90.00 percentile latency (ns)": 2394116, "90th percentile latency (ns)": 2394116, "95.00 percentile latency (ns)": 2891867, "97.00 percentile latency (ns)": 3256167, "99.00 percentile latency (ns)": 4250924, "99.90 percentile latency (ns)": 13507713, "Max latency (ns)": 74718203, "Mean latency (ns)": 1834437, "Min duration satisfied": "Yes", "Min latency (ns)": 1275382, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 543.92, "QPS w/o loadgen overhead": 545.13, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 42.384, "characteristics.good": 21192, "characteristics.power": 19.539930000000005, "characteristics.power.normalized_per_core": 2.4424912500000007, "characteristics.power.normalized_per_processor": 19.539930000000005, "characteristics.samples_per_second": 545.13, "characteristics.samples_per_second.normalized_per_core": 68.14125, "characteristics.samples_per_second.normalized_per_processor": 545.13, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "3bf6ac94bf90e1b4", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 16724110, "90.00 percentile latency (ns)": 22841394, "90th percentile latency (ns)": 22841394, "95.00 percentile latency (ns)": 24206036, "97.00 percentile latency (ns)": 25484979, "99.00 percentile latency (ns)": 31510818, "99.90 percentile latency (ns)": 48167236, "Max latency (ns)": 144601871, "Mean latency (ns)": 18074075, "Min duration satisfied": "Yes", "Min latency (ns)": 13240963, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 55.26, "QPS w/o loadgen overhead": 55.33, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.012, "characteristics.good": 37506, "characteristics.power": 26.151733333333354, "characteristics.power.normalized_per_core": 3.2689666666666692, "characteristics.power.normalized_per_processor": 26.151733333333354, "characteristics.samples_per_second": 55.33, "characteristics.samples_per_second.normalized_per_core": 6.91625, "characteristics.samples_per_second.normalized_per_processor": 55.33, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.3-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.3-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.3_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 58.8235, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "e146dff5f4adc96f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4900052, "90.00 percentile latency (ns)": 7167834, "90th percentile latency (ns)": 7167834, "95.00 percentile latency (ns)": 9021415, "97.00 percentile latency (ns)": 10341125, "99.00 percentile latency (ns)": 12514946, "99.90 percentile latency (ns)": 28193197, "Max latency (ns)": 84466521, "Mean latency (ns)": 5604654, "Min duration satisfied": "Yes", "Min latency (ns)": 4203000, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 178.12, "QPS w/o loadgen overhead": 178.42, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.486, "characteristics.good": 32243, "characteristics.power": 20.940526666666663, "characteristics.power.normalized_per_core": 2.617565833333333, "characteristics.power.normalized_per_processor": 20.940526666666663, "characteristics.samples_per_second": 178.42, "characteristics.samples_per_second.normalized_per_core": 22.3025, "characteristics.samples_per_second.normalized_per_processor": 178.42, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-small_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "71f439883fd916c1", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 23141503, "90.00 percentile latency (ns)": 31213763, "90th percentile latency (ns)": 31213763, "95.00 percentile latency (ns)": 33347777, "97.00 percentile latency (ns)": 34967082, "99.00 percentile latency (ns)": 40400819, "99.90 percentile latency (ns)": 65093580, "Max latency (ns)": 159535848, "Mean latency (ns)": 24855275, "Min duration satisfied": "Yes", "Min latency (ns)": 19252331, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 40.2, "QPS w/o loadgen overhead": 40.23, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 77.542, "characteristics.good": 38771, "characteristics.power": 26.790415973377716, "characteristics.power.normalized_per_core": 3.3488019966722145, "characteristics.power.normalized_per_processor": 26.790415973377716, "characteristics.samples_per_second": 40.23, "characteristics.samples_per_second.normalized_per_core": 5.02875, "characteristics.samples_per_second.normalized_per_processor": 40.23, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite2-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "c3fbe2abdca0d2e9", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2690970, "90.00 percentile latency (ns)": 3947089, "90th percentile latency (ns)": 3947089, "95.00 percentile latency (ns)": 4564684, "97.00 percentile latency (ns)": 5205895, "99.00 percentile latency (ns)": 7363082, "99.90 percentile latency (ns)": 19382632, "Max latency (ns)": 71899824, "Mean latency (ns)": 3054344, "Min duration satisfied": "Yes", "Min latency (ns)": 2348265, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 326.88, "QPS w/o loadgen overhead": 327.4, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.75, "characteristics.good": 28375, "characteristics.power": 19.90226666666664, "characteristics.power.normalized_per_core": 2.48778333333333, "characteristics.power.normalized_per_processor": 19.90226666666664, "characteristics.samples_per_second": 327.4, "characteristics.samples_per_second.normalized_per_core": 40.925, "characteristics.samples_per_second.normalized_per_processor": 327.4, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "f41513b442b6b21b", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3709031, "90.00 percentile latency (ns)": 5334928, "90th percentile latency (ns)": 5334928, "95.00 percentile latency (ns)": 6123919, "97.00 percentile latency (ns)": 7181475, "99.00 percentile latency (ns)": 10110403, "99.90 percentile latency (ns)": 23339836, "Max latency (ns)": 132130634, "Mean latency (ns)": 4216725, "Min duration satisfied": "Yes", "Min latency (ns)": 3105034, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 236.84, "QPS w/o loadgen overhead": 237.15, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 57.002, "characteristics.good": 28501, "characteristics.power": 19.86164999999999, "characteristics.power.normalized_per_core": 2.482706249999999, "characteristics.power.normalized_per_processor": 19.86164999999999, "characteristics.samples_per_second": 237.15, "characteristics.samples_per_second.normalized_per_core": 29.64375, "characteristics.samples_per_second.normalized_per_processor": 237.15, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "4638bb10c82a1d2e", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 15464923, "90.00 percentile latency (ns)": 21046135, "90th percentile latency (ns)": 21046135, "95.00 percentile latency (ns)": 22797987, "97.00 percentile latency (ns)": 24176162, "99.00 percentile latency (ns)": 29478900, "99.90 percentile latency (ns)": 53922267, "Max latency (ns)": 157541099, "Mean latency (ns)": 16572887, "Min duration satisfied": "Yes", "Min latency (ns)": 13119439, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 60.28, "QPS w/o loadgen overhead": 60.34, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.69, "characteristics.good": 37845, "characteristics.power": 26.958799999999986, "characteristics.power.normalized_per_core": 3.3698499999999982, "characteristics.power.normalized_per_processor": 26.958799999999986, "characteristics.samples_per_second": 60.34, "characteristics.samples_per_second.normalized_per_core": 7.5425, "characteristics.samples_per_second.normalized_per_processor": 60.34, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.4-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.4-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.4_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 58.8235, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "aff854dcfa55ebad", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 1961458, "90.00 percentile latency (ns)": 3135782, "90th percentile latency (ns)": 3135782, "95.00 percentile latency (ns)": 4015116, "97.00 percentile latency (ns)": 4495677, "99.00 percentile latency (ns)": 6743458, "99.90 percentile latency (ns)": 19423306, "Max latency (ns)": 108674570, "Mean latency (ns)": 2311563, "Min duration satisfied": "Yes", "Min latency (ns)": 1575392, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 430.61, "QPS w/o loadgen overhead": 432.61, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 52.446, "characteristics.good": 26223, "characteristics.power": 22.86214833333331, "characteristics.power.normalized_per_core": 2.8577685416666636, "characteristics.power.normalized_per_processor": 22.86214833333331, "characteristics.samples_per_second": 432.61, "characteristics.samples_per_second.normalized_per_core": 54.07625, "characteristics.samples_per_second.normalized_per_processor": 432.61, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "9ab3dab080524bca", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8329839, "90.00 percentile latency (ns)": 10870741, "90th percentile latency (ns)": 10870741, "95.00 percentile latency (ns)": 13082354, "97.00 percentile latency (ns)": 14579438, "99.00 percentile latency (ns)": 17455318, "99.90 percentile latency (ns)": 35811343, "Max latency (ns)": 116077835, "Mean latency (ns)": 9040099, "Min duration satisfied": "Yes", "Min latency (ns)": 7282705, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 110.41, "QPS w/o loadgen overhead": 110.62, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.724, "characteristics.good": 36862, "characteristics.power": 24.869499999999988, "characteristics.power.normalized_per_core": 3.1086874999999985, "characteristics.power.normalized_per_processor": 24.869499999999988, "characteristics.samples_per_second": 110.62, "characteristics.samples_per_second.normalized_per_core": 13.8275, "characteristics.samples_per_second.normalized_per_processor": 110.62, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-large_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "4199909be17f4bcf", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4590835, "90.00 percentile latency (ns)": 6853820, "90th percentile latency (ns)": 6853820, "95.00 percentile latency (ns)": 8131926, "97.00 percentile latency (ns)": 9103555, "99.00 percentile latency (ns)": 11757852, "99.90 percentile latency (ns)": 29801438, "Max latency (ns)": 91455473, "Mean latency (ns)": 5295949, "Min duration satisfied": "Yes", "Min latency (ns)": 4040025, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 188.21, "QPS w/o loadgen overhead": 188.82, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.604, "characteristics.good": 33302, "characteristics.power": 26.664799999999985, "characteristics.power.normalized_per_core": 3.333099999999998, "characteristics.power.normalized_per_processor": 26.664799999999985, "characteristics.samples_per_second": 188.82, "characteristics.samples_per_second.normalized_per_core": 23.6025, "characteristics.samples_per_second.normalized_per_processor": 188.82, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "54713a3250a431bf", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7536536, "90.00 percentile latency (ns)": 10020328, "90th percentile latency (ns)": 10020328, "95.00 percentile latency (ns)": 11864028, "97.00 percentile latency (ns)": 13036786, "99.00 percentile latency (ns)": 15834801, "99.90 percentile latency (ns)": 34042350, "Max latency (ns)": 81855822, "Mean latency (ns)": 8146372, "Min duration satisfied": "Yes", "Min latency (ns)": 6472263, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 122.46, "QPS w/o loadgen overhead": 122.75, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.106, "characteristics.good": 34553, "characteristics.power": 26.349333333333334, "characteristics.power.normalized_per_core": 3.2936666666666667, "characteristics.power.normalized_per_processor": 26.349333333333334, "characteristics.samples_per_second": 122.75, "characteristics.samples_per_second.normalized_per_core": 15.34375, "characteristics.samples_per_second.normalized_per_processor": 122.75, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "f4b67b1307d66956", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3129905, "90.00 percentile latency (ns)": 5096845, "90th percentile latency (ns)": 5096845, "95.00 percentile latency (ns)": 6165277, "97.00 percentile latency (ns)": 7178828, "99.00 percentile latency (ns)": 9902378, "99.90 percentile latency (ns)": 27056758, "Max latency (ns)": 118466272, "Mean latency (ns)": 3718475, "Min duration satisfied": "Yes", "Min latency (ns)": 2310763, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 267.97, "QPS w/o loadgen overhead": 268.93, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.822, "characteristics.good": 29911, "characteristics.power": 22.819668333333343, "characteristics.power.normalized_per_core": 2.852458541666668, "characteristics.power.normalized_per_processor": 22.819668333333343, "characteristics.samples_per_second": 268.93, "characteristics.samples_per_second.normalized_per_core": 33.61625, "characteristics.samples_per_second.normalized_per_processor": 268.93, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "46fe514087bcff2a", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4726689, "90.00 percentile latency (ns)": 6070641, "90th percentile latency (ns)": 6070641, "95.00 percentile latency (ns)": 7117619, "97.00 percentile latency (ns)": 8207291, "99.00 percentile latency (ns)": 11350114, "99.90 percentile latency (ns)": 27219152, "Max latency (ns)": 122064324, "Mean latency (ns)": 5174687, "Min duration satisfied": "Yes", "Min latency (ns)": 4133674, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 192.67, "QPS w/o loadgen overhead": 193.25, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.336, "characteristics.good": 34168, "characteristics.power": 23.560900000000014, "characteristics.power.normalized_per_core": 2.945112500000002, "characteristics.power.normalized_per_processor": 23.560900000000014, "characteristics.samples_per_second": 193.25, "characteristics.samples_per_second.normalized_per_core": 24.15625, "characteristics.samples_per_second.normalized_per_processor": 193.25, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-small_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "4c7416baba69f6c8", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7184235, "90.00 percentile latency (ns)": 11062079, "90th percentile latency (ns)": 11062079, "95.00 percentile latency (ns)": 12490657, "97.00 percentile latency (ns)": 13423082, "99.00 percentile latency (ns)": 16395381, "99.90 percentile latency (ns)": 34629373, "Max latency (ns)": 126881056, "Mean latency (ns)": 8071022, "Min duration satisfied": "Yes", "Min latency (ns)": 5572256, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 123.65, "QPS w/o loadgen overhead": 123.9, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.834, "characteristics.good": 34417, "characteristics.power": 24.91740000000001, "characteristics.power.normalized_per_core": 3.1146750000000014, "characteristics.power.normalized_per_processor": 24.91740000000001, "characteristics.samples_per_second": 123.9, "characteristics.samples_per_second.normalized_per_core": 15.4875, "characteristics.samples_per_second.normalized_per_processor": 123.9, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "d930848899a7f448", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 15548639, "90.00 percentile latency (ns)": 22814981, "90th percentile latency (ns)": 22814981, "95.00 percentile latency (ns)": 26046171, "97.00 percentile latency (ns)": 28580587, "99.00 percentile latency (ns)": 32060569, "99.90 percentile latency (ns)": 53223086, "Max latency (ns)": 116097645, "Mean latency (ns)": 17228375, "Min duration satisfied": "Yes", "Min latency (ns)": 12672201, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 57.99, "QPS w/o loadgen overhead": 58.04, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.19, "characteristics.good": 35595, "characteristics.power": 20.895964999999986, "characteristics.power.normalized_per_core": 2.6119956249999983, "characteristics.power.normalized_per_processor": 20.895964999999986, "characteristics.samples_per_second": 58.04, "characteristics.samples_per_second.normalized_per_core": 7.255, "characteristics.samples_per_second.normalized_per_processor": 58.04, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite0-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 52.6316, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "04e239a1499e9049", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 37212736, "90.00 percentile latency (ns)": 47939220, "90th percentile latency (ns)": 47939220, "95.00 percentile latency (ns)": 50026309, "97.00 percentile latency (ns)": 51663493, "99.00 percentile latency (ns)": 59581559, "99.90 percentile latency (ns)": 112591195, "Max latency (ns)": 180817905, "Mean latency (ns)": 39322589, "Min duration satisfied": "Yes", "Min latency (ns)": 29585939, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 25.42, "QPS w/o loadgen overhead": 25.43, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 79.268, "characteristics.good": 39634, "characteristics.power": 27.224866666666678, "characteristics.power.normalized_per_core": 3.4031083333333347, "characteristics.power.normalized_per_processor": 27.224866666666678, "characteristics.samples_per_second": 25.43, "characteristics.samples_per_second.normalized_per_core": 3.17875, "characteristics.samples_per_second.normalized_per_processor": 25.43, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite3-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 28.5714, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "cf14131bbb8c68d5", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4405241, "90.00 percentile latency (ns)": 5623270, "90th percentile latency (ns)": 5623270, "95.00 percentile latency (ns)": 6657477, "97.00 percentile latency (ns)": 7591962, "99.00 percentile latency (ns)": 10811891, "99.90 percentile latency (ns)": 26974855, "Max latency (ns)": 107788566, "Mean latency (ns)": 4837098, "Min duration satisfied": "Yes", "Min latency (ns)": 3874619, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 206.1, "QPS w/o loadgen overhead": 206.74, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.216, "characteristics.good": 33108, "characteristics.power": 22.74788333333334, "characteristics.power.normalized_per_core": 2.8434854166666677, "characteristics.power.normalized_per_processor": 22.74788333333334, "characteristics.samples_per_second": 206.74, "characteristics.samples_per_second.normalized_per_core": 25.8425, "characteristics.samples_per_second.normalized_per_processor": 206.74, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-small_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "d2045d17f149419f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 48000961, "90.00 percentile latency (ns)": 59108080, "90th percentile latency (ns)": 59108080, "95.00 percentile latency (ns)": 61944471, "97.00 percentile latency (ns)": 64115036, "99.00 percentile latency (ns)": 71086374, "99.90 percentile latency (ns)": 123854891, "Max latency (ns)": 169733581, "Mean latency (ns)": 49883475, "Min duration satisfied": "Yes", "Min latency (ns)": 39359594, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.04, "QPS w/o loadgen overhead": 20.05, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.344, "characteristics.good": 38172, "characteristics.power": 20.96996505823627, "characteristics.power.normalized_per_core": 2.6212456322795337, "characteristics.power.normalized_per_processor": 20.96996505823627, "characteristics.samples_per_second": 20.05, "characteristics.samples_per_second.normalized_per_core": 2.50625, "characteristics.samples_per_second.normalized_per_processor": 20.05, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite3-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 20, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "af4732a657d678a0", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3630537, "90.00 percentile latency (ns)": 5116846, "90th percentile latency (ns)": 5116846, "95.00 percentile latency (ns)": 5861774, "97.00 percentile latency (ns)": 7152139, "99.00 percentile latency (ns)": 10289950, "99.90 percentile latency (ns)": 22474325, "Max latency (ns)": 107298300, "Mean latency (ns)": 4095315, "Min duration satisfied": "Yes", "Min latency (ns)": 3041709, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 243.82, "QPS w/o loadgen overhead": 244.18, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 54.568, "characteristics.good": 27284, "characteristics.power": 19.937760000000008, "characteristics.power.normalized_per_core": 2.492220000000001, "characteristics.power.normalized_per_processor": 19.937760000000008, "characteristics.samples_per_second": 244.18, "characteristics.samples_per_second.normalized_per_core": 30.5225, "characteristics.samples_per_second.normalized_per_processor": 244.18, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "d749742e59bb9054", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2501413, "90.00 percentile latency (ns)": 3277252, "90th percentile latency (ns)": 3277252, "95.00 percentile latency (ns)": 3731992, "97.00 percentile latency (ns)": 4267656, "99.00 percentile latency (ns)": 5986151, "99.90 percentile latency (ns)": 17350855, "Max latency (ns)": 86733421, "Mean latency (ns)": 2730931, "Min duration satisfied": "Yes", "Min latency (ns)": 2120819, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 365.22, "QPS w/o loadgen overhead": 366.18, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.478, "characteristics.good": 24739, "characteristics.power": 22.56965833333335, "characteristics.power.normalized_per_core": 2.821207291666669, "characteristics.power.normalized_per_processor": 22.56965833333335, "characteristics.samples_per_second": 366.18, "characteristics.samples_per_second.normalized_per_core": 45.7725, "characteristics.samples_per_second.normalized_per_processor": 366.18, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "83fb3dfa47168e3e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10585532, "90.00 percentile latency (ns)": 12770514, "90th percentile latency (ns)": 12770514, "95.00 percentile latency (ns)": 15206411, "97.00 percentile latency (ns)": 16584117, "99.00 percentile latency (ns)": 19307427, "99.90 percentile latency (ns)": 40651492, "Max latency (ns)": 92449624, "Mean latency (ns)": 11226998, "Min duration satisfied": "Yes", "Min latency (ns)": 9133625, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 88.99, "QPS w/o loadgen overhead": 89.07, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.126, "characteristics.good": 34063, "characteristics.power": 20.450199999999988, "characteristics.power.normalized_per_core": 2.5562749999999985, "characteristics.power.normalized_per_processor": 20.450199999999988, "characteristics.samples_per_second": 89.07, "characteristics.samples_per_second.normalized_per_core": 11.13375, "characteristics.samples_per_second.normalized_per_processor": 89.07, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "45bea004e45fd8d0", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10797839, "90.00 percentile latency (ns)": 15940268, "90th percentile latency (ns)": 15940268, "95.00 percentile latency (ns)": 17383854, "97.00 percentile latency (ns)": 18298840, "99.00 percentile latency (ns)": 22575078, "99.90 percentile latency (ns)": 40797362, "Max latency (ns)": 142581766, "Mean latency (ns)": 11984913, "Min duration satisfied": "Yes", "Min latency (ns)": 8960269, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 83.32, "QPS w/o loadgen overhead": 83.44, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.76, "characteristics.good": 37880, "characteristics.power": 25.089066666666636, "characteristics.power.normalized_per_core": 3.1361333333333294, "characteristics.power.normalized_per_processor": 25.089066666666636, "characteristics.samples_per_second": 83.44, "characteristics.samples_per_second.normalized_per_core": 10.43, "characteristics.samples_per_second.normalized_per_processor": 83.44, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-large_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "c6f04802ba625c00", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5652875, "90.00 percentile latency (ns)": 7659301, "90th percentile latency (ns)": 7659301, "95.00 percentile latency (ns)": 9181394, "97.00 percentile latency (ns)": 10583316, "99.00 percentile latency (ns)": 13377745, "99.90 percentile latency (ns)": 29638953, "Max latency (ns)": 125500503, "Mean latency (ns)": 6267894, "Min duration satisfied": "Yes", "Min latency (ns)": 4845056, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 159.14, "QPS w/o loadgen overhead": 159.54, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 65.75, "characteristics.good": 32875, "characteristics.power": 24.015316666666674, "characteristics.power.normalized_per_core": 3.0019145833333343, "characteristics.power.normalized_per_processor": 24.015316666666674, "characteristics.samples_per_second": 159.54, "characteristics.samples_per_second.normalized_per_core": 19.9425, "characteristics.samples_per_second.normalized_per_processor": 159.54, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "cbd724619832517b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7005742, "90.00 percentile latency (ns)": 9184729, "90th percentile latency (ns)": 9184729, "95.00 percentile latency (ns)": 11226168, "97.00 percentile latency (ns)": 12639710, "99.00 percentile latency (ns)": 15337657, "99.90 percentile latency (ns)": 33539216, "Max latency (ns)": 79155877, "Mean latency (ns)": 7634506, "Min duration satisfied": "Yes", "Min latency (ns)": 5847607, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 130.79, "QPS w/o loadgen overhead": 130.98, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.768, "characteristics.good": 32384, "characteristics.power": 20.358194999999995, "characteristics.power.normalized_per_core": 2.5447743749999994, "characteristics.power.normalized_per_processor": 20.358194999999995, "characteristics.samples_per_second": 130.98, "characteristics.samples_per_second.normalized_per_core": 16.3725, "characteristics.samples_per_second.normalized_per_processor": 130.98, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "dad8661148f38813", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9022807, "90.00 percentile latency (ns)": 13695322, "90th percentile latency (ns)": 13695322, "95.00 percentile latency (ns)": 15096466, "97.00 percentile latency (ns)": 15996122, "99.00 percentile latency (ns)": 20128454, "99.90 percentile latency (ns)": 37414771, "Max latency (ns)": 133446223, "Mean latency (ns)": 10139688, "Min duration satisfied": "Yes", "Min latency (ns)": 7265133, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 98.45, "QPS w/o loadgen overhead": 98.62, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.122, "characteristics.good": 35061, "characteristics.power": 24.795099999999987, "characteristics.power.normalized_per_core": 3.0993874999999984, "characteristics.power.normalized_per_processor": 24.795099999999987, "characteristics.samples_per_second": 98.62, "characteristics.samples_per_second.normalized_per_core": 12.3275, "characteristics.samples_per_second.normalized_per_processor": 98.62, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "62e4e4952b415213", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6292066, "90.00 percentile latency (ns)": 8373174, "90th percentile latency (ns)": 8373174, "95.00 percentile latency (ns)": 10259658, "97.00 percentile latency (ns)": 11653292, "99.00 percentile latency (ns)": 13798485, "99.90 percentile latency (ns)": 29061929, "Max latency (ns)": 74114720, "Mean latency (ns)": 6968849, "Min duration satisfied": "Yes", "Min latency (ns)": 5517718, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 143.32, "QPS w/o loadgen overhead": 143.5, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.08, "characteristics.good": 32040, "characteristics.power": 20.282493333333353, "characteristics.power.normalized_per_core": 2.535311666666669, "characteristics.power.normalized_per_processor": 20.282493333333353, "characteristics.samples_per_second": 143.5, "characteristics.samples_per_second.normalized_per_core": 17.9375, "characteristics.samples_per_second.normalized_per_processor": 143.5, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "4ce1ba3b7321f525", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5041484, "90.00 percentile latency (ns)": 7466022, "90th percentile latency (ns)": 7466022, "95.00 percentile latency (ns)": 8867814, "97.00 percentile latency (ns)": 9945465, "99.00 percentile latency (ns)": 12317130, "99.90 percentile latency (ns)": 29017584, "Max latency (ns)": 117524865, "Mean latency (ns)": 5673049, "Min duration satisfied": "Yes", "Min latency (ns)": 4022145, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 175.85, "QPS w/o loadgen overhead": 176.27, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.282, "characteristics.good": 32141, "characteristics.power": 23.11268333333333, "characteristics.power.normalized_per_core": 2.889085416666666, "characteristics.power.normalized_per_processor": 23.11268333333333, "characteristics.samples_per_second": 176.27, "characteristics.samples_per_second.normalized_per_core": 22.03375, "characteristics.samples_per_second.normalized_per_processor": 176.27, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "0b6b2b1c700a8914", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8724291, "90.00 percentile latency (ns)": 13120734, "90th percentile latency (ns)": 13120734, "95.00 percentile latency (ns)": 14638053, "97.00 percentile latency (ns)": 15580947, "99.00 percentile latency (ns)": 19400276, "99.90 percentile latency (ns)": 37184911, "Max latency (ns)": 134644753, "Mean latency (ns)": 9739318, "Min duration satisfied": "Yes", "Min latency (ns)": 7056600, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 102.49, "QPS w/o loadgen overhead": 102.68, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.678, "characteristics.good": 36339, "characteristics.power": 24.889233333333312, "characteristics.power.normalized_per_core": 3.111154166666664, "characteristics.power.normalized_per_processor": 24.889233333333312, "characteristics.samples_per_second": 102.68, "characteristics.samples_per_second.normalized_per_core": 12.835, "characteristics.samples_per_second.normalized_per_processor": 102.68, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "cca4a989a1a68952", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6705487, "90.00 percentile latency (ns)": 10261327, "90th percentile latency (ns)": 10261327, "95.00 percentile latency (ns)": 12765666, "97.00 percentile latency (ns)": 13826000, "99.00 percentile latency (ns)": 15787880, "99.90 percentile latency (ns)": 32885455, "Max latency (ns)": 125173117, "Mean latency (ns)": 7633632, "Min duration satisfied": "Yes", "Min latency (ns)": 5751780, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 130.83, "QPS w/o loadgen overhead": 131.0, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.972, "characteristics.good": 32486, "characteristics.power": 20.203508333333346, "characteristics.power.normalized_per_core": 2.5254385416666683, "characteristics.power.normalized_per_processor": 20.203508333333346, "characteristics.samples_per_second": 131.0, "characteristics.samples_per_second.normalized_per_core": 16.375, "characteristics.samples_per_second.normalized_per_processor": 131.0, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "a049daac26eb3346", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11425473, "90.00 percentile latency (ns)": 16676588, "90th percentile latency (ns)": 16676588, "95.00 percentile latency (ns)": 18071019, "97.00 percentile latency (ns)": 19099161, "99.00 percentile latency (ns)": 23715208, "99.90 percentile latency (ns)": 48870165, "Max latency (ns)": 168200761, "Mean latency (ns)": 12604965, "Min duration satisfied": "Yes", "Min latency (ns)": 8956146, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 79.21, "QPS w/o loadgen overhead": 79.33, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.77, "characteristics.good": 35885, "characteristics.power": 25.58968333333335, "characteristics.power.normalized_per_core": 3.198710416666669, "characteristics.power.normalized_per_processor": 25.58968333333335, "characteristics.samples_per_second": 79.33, "characteristics.samples_per_second.normalized_per_core": 9.91625, "characteristics.samples_per_second.normalized_per_processor": 79.33, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "f5a9ba29782d4b3f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5342410, "90.00 percentile latency (ns)": 8618986, "90th percentile latency (ns)": 8618986, "95.00 percentile latency (ns)": 10212894, "97.00 percentile latency (ns)": 11343877, "99.00 percentile latency (ns)": 13594420, "99.90 percentile latency (ns)": 31036232, "Max latency (ns)": 130546550, "Mean latency (ns)": 6256980, "Min duration satisfied": "Yes", "Min latency (ns)": 4293604, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 159.39, "QPS w/o loadgen overhead": 159.82, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.75, "characteristics.good": 33375, "characteristics.power": 24.163016666666653, "characteristics.power.normalized_per_core": 3.0203770833333317, "characteristics.power.normalized_per_processor": 24.163016666666653, "characteristics.samples_per_second": 159.82, "characteristics.samples_per_second.normalized_per_core": 19.9775, "characteristics.samples_per_second.normalized_per_processor": 159.82, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "45158702bef29377", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 1773711, "90.00 percentile latency (ns)": 2650712, "90th percentile latency (ns)": 2650712, "95.00 percentile latency (ns)": 3560000, "97.00 percentile latency (ns)": 3873199, "99.00 percentile latency (ns)": 5433365, "99.90 percentile latency (ns)": 18785197, "Max latency (ns)": 89497696, "Mean latency (ns)": 2034427, "Min duration satisfied": "Yes", "Min latency (ns)": 1345853, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 489.57, "QPS w/o loadgen overhead": 491.54, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 47.14, "characteristics.good": 23570, "characteristics.power": 21.09288833333336, "characteristics.power.normalized_per_core": 2.63661104166667, "characteristics.power.normalized_per_processor": 21.09288833333336, "characteristics.samples_per_second": 491.54, "characteristics.samples_per_second.normalized_per_core": 61.4425, "characteristics.samples_per_second.normalized_per_processor": 491.54, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "50ac8402e8a4a571", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2956035, "90.00 percentile latency (ns)": 4418372, "90th percentile latency (ns)": 4418372, "95.00 percentile latency (ns)": 5054113, "97.00 percentile latency (ns)": 5663004, "99.00 percentile latency (ns)": 8560222, "99.90 percentile latency (ns)": 23244069, "Max latency (ns)": 103654710, "Mean latency (ns)": 3353959, "Min duration satisfied": "Yes", "Min latency (ns)": 2416011, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 296.99, "QPS w/o loadgen overhead": 298.16, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.988, "characteristics.good": 28494, "characteristics.power": 22.857074999999995, "characteristics.power.normalized_per_core": 2.8571343749999993, "characteristics.power.normalized_per_processor": 22.857074999999995, "characteristics.samples_per_second": 298.16, "characteristics.samples_per_second.normalized_per_core": 37.27, "characteristics.samples_per_second.normalized_per_processor": 298.16, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "fb42296506a7a85f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3633383, "90.00 percentile latency (ns)": 5577377, "90th percentile latency (ns)": 5577377, "95.00 percentile latency (ns)": 6650802, "97.00 percentile latency (ns)": 7561692, "99.00 percentile latency (ns)": 9975435, "99.90 percentile latency (ns)": 26043727, "Max latency (ns)": 73759585, "Mean latency (ns)": 4235892, "Min duration satisfied": "Yes", "Min latency (ns)": 3121776, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 235.16, "QPS w/o loadgen overhead": 236.08, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.716, "characteristics.good": 31858, "characteristics.power": 25.518683333333318, "characteristics.power.normalized_per_core": 3.1898354166666647, "characteristics.power.normalized_per_processor": 25.518683333333318, "characteristics.samples_per_second": 236.08, "characteristics.samples_per_second.normalized_per_core": 29.51, "characteristics.samples_per_second.normalized_per_processor": 236.08, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "0e0b040a5bebf5a9", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3368697, "90.00 percentile latency (ns)": 4727221, "90th percentile latency (ns)": 4727221, "95.00 percentile latency (ns)": 5263181, "97.00 percentile latency (ns)": 5827690, "99.00 percentile latency (ns)": 8678150, "99.90 percentile latency (ns)": 22506781, "Max latency (ns)": 117154503, "Mean latency (ns)": 3757035, "Min duration satisfied": "Yes", "Min latency (ns)": 2794846, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 265.7, "QPS w/o loadgen overhead": 266.17, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 55.54, "characteristics.good": 27770, "characteristics.power": 19.78485666666666, "characteristics.power.normalized_per_core": 2.4731070833333324, "characteristics.power.normalized_per_processor": 19.78485666666666, "characteristics.samples_per_second": 266.17, "characteristics.samples_per_second.normalized_per_core": 33.27125, "characteristics.samples_per_second.normalized_per_processor": 266.17, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "86d6268073418bbf", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3849786, "90.00 percentile latency (ns)": 5387489, "90th percentile latency (ns)": 5387489, "95.00 percentile latency (ns)": 6406485, "97.00 percentile latency (ns)": 7427270, "99.00 percentile latency (ns)": 10357904, "99.90 percentile latency (ns)": 25522997, "Max latency (ns)": 136669675, "Mean latency (ns)": 4344322, "Min duration satisfied": "Yes", "Min latency (ns)": 3260443, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 229.44, "QPS w/o loadgen overhead": 230.19, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.322, "characteristics.good": 29661, "characteristics.power": 22.761684999999996, "characteristics.power.normalized_per_core": 2.8452106249999995, "characteristics.power.normalized_per_processor": 22.761684999999996, "characteristics.samples_per_second": 230.19, "characteristics.samples_per_second.normalized_per_core": 28.77375, "characteristics.samples_per_second.normalized_per_processor": 230.19, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "e659c293c5007324", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 30469794, "90.00 percentile latency (ns)": 104486530, "90th percentile latency (ns)": 104486530, "95.00 percentile latency (ns)": 120078260, "97.00 percentile latency (ns)": 124319548, "99.00 percentile latency (ns)": 131195289, "99.90 percentile latency (ns)": 140987109, "Max latency (ns)": 157011839, "Mean latency (ns)": 45263185, "Min duration satisfied": "Yes", "Min latency (ns)": 29046462, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 22.07, "QPS w/o loadgen overhead": 22.09, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.242, "characteristics.good": 34121, "characteristics.samples_per_second": 22.09, "characteristics.samples_per_second.normalized_per_core": 3.6816666666666666, "characteristics.samples_per_second.normalized_per_processor": 22.09, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.5185, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "27905d409c45c08f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 40567550, "90.00 percentile latency (ns)": 136545072, "90th percentile latency (ns)": 136545072, "95.00 percentile latency (ns)": 193013373, "97.00 percentile latency (ns)": 202012504, "99.00 percentile latency (ns)": 210668304, "99.90 percentile latency (ns)": 220430705, "Max latency (ns)": 227976863, "Mean latency (ns)": 61607395, "Min duration satisfied": "Yes", "Min latency (ns)": 37231459, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.23, "QPS w/o loadgen overhead": 16.23, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.738, "characteristics.good": 35369, "characteristics.samples_per_second": 16.23, "characteristics.samples_per_second.normalized_per_core": 2.705, "characteristics.samples_per_second.normalized_per_processor": 16.23, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.2857, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "ca240fab286f4b59", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 324260412, "90.00 percentile latency (ns)": 416209128, "90th percentile latency (ns)": 416209128, "95.00 percentile latency (ns)": 433856470, "97.00 percentile latency (ns)": 440011178, "99.00 percentile latency (ns)": 455279870, "99.90 percentile latency (ns)": 473264667, "Max latency (ns)": 476895076, "Mean latency (ns)": 305669961, "Min duration satisfied": "Yes", "Min latency (ns)": 178695293, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 3.27, "QPS w/o loadgen overhead": 3.27, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 78.514, "characteristics.good": 39257, "characteristics.samples_per_second": 3.27, "characteristics.samples_per_second.normalized_per_core": 0.545, "characteristics.samples_per_second.normalized_per_processor": 3.27, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite4-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.3557, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "873fadb4ed98bb14", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 13528391, "90.00 percentile latency (ns)": 19582815, "90th percentile latency (ns)": 19582815, "95.00 percentile latency (ns)": 35533208, "97.00 percentile latency (ns)": 54296231, "99.00 percentile latency (ns)": 149126253, "99.90 percentile latency (ns)": 185098806, "Max latency (ns)": 202497228, "Mean latency (ns)": 18015005, "Min duration satisfied": "Yes", "Min latency (ns)": 13010682, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 55.37, "QPS w/o loadgen overhead": 55.51, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.542, "characteristics.good": 31771, "characteristics.samples_per_second": 55.51, "characteristics.samples_per_second.normalized_per_core": 9.251666666666667, "characteristics.samples_per_second.normalized_per_processor": 55.51, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 38.4615, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "54c35800bf7eb61b", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10213005, "90.00 percentile latency (ns)": 71072321, "90th percentile latency (ns)": 71072321, "95.00 percentile latency (ns)": 122850958, "97.00 percentile latency (ns)": 149072712, "99.00 percentile latency (ns)": 165629398, "99.90 percentile latency (ns)": 178656305, "Max latency (ns)": 204399481, "Mean latency (ns)": 24784573, "Min duration satisfied": "Yes", "Min latency (ns)": 8405253, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 40.28, "QPS w/o loadgen overhead": 40.35, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 52.172, "characteristics.good": 26086, "characteristics.samples_per_second": 40.35, "characteristics.samples_per_second.normalized_per_core": 6.7250000000000005, "characteristics.samples_per_second.normalized_per_processor": 40.35, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 38.4615, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "bc9f2db66a116362", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 19576646, "90.00 percentile latency (ns)": 152375822, "90th percentile latency (ns)": 152375822, "95.00 percentile latency (ns)": 168325695, "97.00 percentile latency (ns)": 173458124, "99.00 percentile latency (ns)": 180401950, "99.90 percentile latency (ns)": 193479425, "Max latency (ns)": 207864628, "Mean latency (ns)": 50274458, "Min duration satisfied": "Yes", "Min latency (ns)": 10450992, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 19.87, "QPS w/o loadgen overhead": 19.89, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.402, "characteristics.good": 29201, "characteristics.samples_per_second": 19.89, "characteristics.samples_per_second.normalized_per_core": 3.315, "characteristics.samples_per_second.normalized_per_processor": 19.89, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 16.3934, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "12d67df13442afa2", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 21261351, "90.00 percentile latency (ns)": 118634059, "90th percentile latency (ns)": 118634059, "95.00 percentile latency (ns)": 172336421, "97.00 percentile latency (ns)": 180496403, "99.00 percentile latency (ns)": 188827033, "99.90 percentile latency (ns)": 199542401, "Max latency (ns)": 209096039, "Mean latency (ns)": 42982582, "Min duration satisfied": "Yes", "Min latency (ns)": 19823441, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 23.25, "QPS w/o loadgen overhead": 23.27, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 60.976, "characteristics.good": 30488, "characteristics.samples_per_second": 23.27, "characteristics.samples_per_second.normalized_per_core": 3.8783333333333334, "characteristics.samples_per_second.normalized_per_processor": 23.27, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.873, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "4e4d728a6f27c432", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4947540, "90.00 percentile latency (ns)": 5219082, "90th percentile latency (ns)": 5219082, "95.00 percentile latency (ns)": 6182164, "97.00 percentile latency (ns)": 7044331, "99.00 percentile latency (ns)": 29754655, "99.90 percentile latency (ns)": 92626258, "Max latency (ns)": 107109913, "Mean latency (ns)": 5504405, "Min duration satisfied": "Yes", "Min latency (ns)": 3904832, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 180.71, "QPS w/o loadgen overhead": 181.67, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 48.162, "characteristics.good": 24081, "characteristics.samples_per_second": 181.67, "characteristics.samples_per_second.normalized_per_core": 30.278333333333332, "characteristics.samples_per_second.normalized_per_processor": 181.67, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "b40e9a4f4504cd90", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 74152147, "90.00 percentile latency (ns)": 149194463, "90th percentile latency (ns)": 149194463, "95.00 percentile latency (ns)": 196076656, "97.00 percentile latency (ns)": 205041062, "99.00 percentile latency (ns)": 214697846, "99.90 percentile latency (ns)": 223914862, "Max latency (ns)": 228916589, "Mean latency (ns)": 87843985, "Min duration satisfied": "Yes", "Min latency (ns)": 63649237, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 11.38, "QPS w/o loadgen overhead": 11.38, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.068, "characteristics.good": 37534, "characteristics.samples_per_second": 11.38, "characteristics.samples_per_second.normalized_per_core": 1.8966666666666667, "characteristics.samples_per_second.normalized_per_processor": 11.38, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite0-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 10.8696, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "84195b708a28e189", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5445188, "90.00 percentile latency (ns)": 6685614, "90th percentile latency (ns)": 6685614, "95.00 percentile latency (ns)": 9236543, "97.00 percentile latency (ns)": 22088027, "99.00 percentile latency (ns)": 80195437, "99.90 percentile latency (ns)": 98101453, "Max latency (ns)": 110506728, "Mean latency (ns)": 7363468, "Min duration satisfied": "Yes", "Min latency (ns)": 4988116, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 135.23, "QPS w/o loadgen overhead": 135.81, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 50.242, "characteristics.good": 25121, "characteristics.samples_per_second": 135.81, "characteristics.samples_per_second.normalized_per_core": 22.635, "characteristics.samples_per_second.normalized_per_processor": 135.81, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "5a8f0b8b9d1363b1", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 19889070, "90.00 percentile latency (ns)": 100591482, "90th percentile latency (ns)": 100591482, "95.00 percentile latency (ns)": 104656294, "97.00 percentile latency (ns)": 106799125, "99.00 percentile latency (ns)": 110147798, "99.90 percentile latency (ns)": 116380992, "Max latency (ns)": 121208672, "Mean latency (ns)": 40383670, "Min duration satisfied": "Yes", "Min latency (ns)": 12801268, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 24.73, "QPS w/o loadgen overhead": 24.76, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 60.842, "characteristics.good": 30421, "characteristics.samples_per_second": 24.76, "characteristics.samples_per_second.normalized_per_core": 4.126666666666667, "characteristics.samples_per_second.normalized_per_processor": 24.76, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.8679, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "503f58434ab209df", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 29477284, "90.00 percentile latency (ns)": 112832957, "90th percentile latency (ns)": 112832957, "95.00 percentile latency (ns)": 179696685, "97.00 percentile latency (ns)": 192010920, "99.00 percentile latency (ns)": 202799284, "99.90 percentile latency (ns)": 211760180, "Max latency (ns)": 218109470, "Mean latency (ns)": 49641921, "Min duration satisfied": "Yes", "Min latency (ns)": 28520618, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.13, "QPS w/o loadgen overhead": 20.14, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.016, "characteristics.good": 34508, "characteristics.samples_per_second": 20.14, "characteristics.samples_per_second.normalized_per_core": 3.356666666666667, "characteristics.samples_per_second.normalized_per_processor": 20.14, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 20.4082, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "6db4fb4254ebea64", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 58536162, "90.00 percentile latency (ns)": 100737892, "90th percentile latency (ns)": 100737892, "95.00 percentile latency (ns)": 168624029, "97.00 percentile latency (ns)": 212149782, "99.00 percentile latency (ns)": 242464318, "99.90 percentile latency (ns)": 265273246, "Max latency (ns)": 282413735, "Mean latency (ns)": 72438353, "Min duration satisfied": "Yes", "Min latency (ns)": 53482196, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 13.8, "QPS w/o loadgen overhead": 13.8, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.098, "characteristics.good": 37049, "characteristics.samples_per_second": 13.8, "characteristics.samples_per_second.normalized_per_core": 2.3000000000000003, "characteristics.samples_per_second.normalized_per_processor": 13.8, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite1-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.987, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "e6f466282308e819", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11609135, "90.00 percentile latency (ns)": 13368499, "90th percentile latency (ns)": 13368499, "95.00 percentile latency (ns)": 22314571, "97.00 percentile latency (ns)": 33863360, "99.00 percentile latency (ns)": 97990265, "99.90 percentile latency (ns)": 182462430, "Max latency (ns)": 202069877, "Mean latency (ns)": 13921083, "Min duration satisfied": "Yes", "Min latency (ns)": 10051966, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 71.65, "QPS w/o loadgen overhead": 71.83, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.924, "characteristics.good": 30962, "characteristics.samples_per_second": 71.83, "characteristics.samples_per_second.normalized_per_core": 11.971666666666666, "characteristics.samples_per_second.normalized_per_processor": 71.83, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "6447fb9d70d4bc3e", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10826962, "90.00 percentile latency (ns)": 16691362, "90th percentile latency (ns)": 16691362, "95.00 percentile latency (ns)": 34728431, "97.00 percentile latency (ns)": 75910651, "99.00 percentile latency (ns)": 102874204, "99.90 percentile latency (ns)": 112973233, "Max latency (ns)": 122977561, "Mean latency (ns)": 15088005, "Min duration satisfied": "Yes", "Min latency (ns)": 10338047, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 66.03, "QPS w/o loadgen overhead": 66.28, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 62.298, "characteristics.good": 31149, "characteristics.samples_per_second": 66.28, "characteristics.samples_per_second.normalized_per_core": 11.046666666666667, "characteristics.samples_per_second.normalized_per_processor": 66.28, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 62.5, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "2e643e187a7b923f", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5205994, "90.00 percentile latency (ns)": 5548734, "90th percentile latency (ns)": 5548734, "95.00 percentile latency (ns)": 6875236, "97.00 percentile latency (ns)": 7661844, "99.00 percentile latency (ns)": 27438555, "99.90 percentile latency (ns)": 139583853, "Max latency (ns)": 189544897, "Mean latency (ns)": 5910820, "Min duration satisfied": "Yes", "Min latency (ns)": 4503670, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 168.24, "QPS w/o loadgen overhead": 169.18, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.12, "characteristics.good": 24560, "characteristics.samples_per_second": 169.18, "characteristics.samples_per_second.normalized_per_core": 28.19666666666667, "characteristics.samples_per_second.normalized_per_processor": 169.18, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "8259fa465aa73c49", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 40968398, "90.00 percentile latency (ns)": 69597610, "90th percentile latency (ns)": 69597610, "95.00 percentile latency (ns)": 123156810, "97.00 percentile latency (ns)": 173083050, "99.00 percentile latency (ns)": 210164310, "99.90 percentile latency (ns)": 229219462, "Max latency (ns)": 254823430, "Mean latency (ns)": 52952633, "Min duration satisfied": "Yes", "Min latency (ns)": 39743381, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 18.87, "QPS w/o loadgen overhead": 18.88, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.584, "characteristics.good": 36792, "characteristics.samples_per_second": 18.88, "characteristics.samples_per_second.normalized_per_core": 3.1466666666666665, "characteristics.samples_per_second.normalized_per_processor": 18.88, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.4-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.4-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_140.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.1818, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "f3e074d53a43dd3b", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 29471510, "90.00 percentile latency (ns)": 59180448, "90th percentile latency (ns)": 59180448, "95.00 percentile latency (ns)": 111633316, "97.00 percentile latency (ns)": 120648137, "99.00 percentile latency (ns)": 127563838, "99.90 percentile latency (ns)": 138155778, "Max latency (ns)": 144373010, "Mean latency (ns)": 38957607, "Min duration satisfied": "Yes", "Min latency (ns)": 28517759, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 25.62, "QPS w/o loadgen overhead": 25.67, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.772, "characteristics.good": 35386, "characteristics.samples_per_second": 25.67, "characteristics.samples_per_second.normalized_per_core": 4.278333333333333, "characteristics.samples_per_second.normalized_per_processor": 25.67, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 24.3902, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "6c403cbd71e71584", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10213590, "90.00 percentile latency (ns)": 78457510, "90th percentile latency (ns)": 78457510, "95.00 percentile latency (ns)": 87635098, "97.00 percentile latency (ns)": 90668724, "99.00 percentile latency (ns)": 95825396, "99.90 percentile latency (ns)": 103955319, "Max latency (ns)": 116438366, "Mean latency (ns)": 25448126, "Min duration satisfied": "Yes", "Min latency (ns)": 6526921, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 39.2, "QPS w/o loadgen overhead": 39.3, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 47.808, "characteristics.good": 23904, "characteristics.samples_per_second": 39.3, "characteristics.samples_per_second.normalized_per_core": 6.55, "characteristics.samples_per_second.normalized_per_processor": 39.3, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 37.037, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "763d4273bdb63119", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 28009350, "90.00 percentile latency (ns)": 163724531, "90th percentile latency (ns)": 163724531, "95.00 percentile latency (ns)": 176483828, "97.00 percentile latency (ns)": 181654451, "99.00 percentile latency (ns)": 188445082, "99.90 percentile latency (ns)": 197133799, "Max latency (ns)": 204222224, "Mean latency (ns)": 58102036, "Min duration satisfied": "Yes", "Min latency (ns)": 16093297, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.2, "QPS w/o loadgen overhead": 17.21, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.038, "characteristics.good": 32019, "characteristics.samples_per_second": 17.21, "characteristics.samples_per_second.normalized_per_core": 2.8683333333333336, "characteristics.samples_per_second.normalized_per_processor": 17.21, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.5, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "22003a3b425889bf", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 15285732, "90.00 percentile latency (ns)": 66472346, "90th percentile latency (ns)": 66472346, "95.00 percentile latency (ns)": 90631699, "97.00 percentile latency (ns)": 95329269, "99.00 percentile latency (ns)": 100950435, "99.90 percentile latency (ns)": 110517228, "Max latency (ns)": 127731957, "Mean latency (ns)": 26353731, "Min duration satisfied": "Yes", "Min latency (ns)": 11648032, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 37.88, "QPS w/o loadgen overhead": 37.95, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 51.848, "characteristics.good": 25924, "characteristics.samples_per_second": 37.95, "characteristics.samples_per_second.normalized_per_core": 6.325, "characteristics.samples_per_second.normalized_per_processor": 37.95, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 21.2766, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "14b92900b63a9cba", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8706610, "90.00 percentile latency (ns)": 9571626, "90th percentile latency (ns)": 9571626, "95.00 percentile latency (ns)": 11733853, "97.00 percentile latency (ns)": 20562377, "99.00 percentile latency (ns)": 50446793, "99.90 percentile latency (ns)": 172257479, "Max latency (ns)": 196912385, "Mean latency (ns)": 10222979, "Min duration satisfied": "Yes", "Min latency (ns)": 7546065, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 97.51, "QPS w/o loadgen overhead": 97.82, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.226, "characteristics.good": 29613, "characteristics.samples_per_second": 97.82, "characteristics.samples_per_second.normalized_per_core": 16.30333333333333, "characteristics.samples_per_second.normalized_per_processor": 97.82, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "daea0d0426abef66", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 16706711, "90.00 percentile latency (ns)": 20832059, "90th percentile latency (ns)": 20832059, "95.00 percentile latency (ns)": 34428515, "97.00 percentile latency (ns)": 45665545, "99.00 percentile latency (ns)": 98353047, "99.90 percentile latency (ns)": 116337492, "Max latency (ns)": 134855826, "Mean latency (ns)": 19576610, "Min duration satisfied": "Yes", "Min latency (ns)": 15677421, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 50.99, "QPS w/o loadgen overhead": 51.08, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.248, "characteristics.good": 34124, "characteristics.samples_per_second": 51.08, "characteristics.samples_per_second.normalized_per_core": 8.513333333333334, "characteristics.samples_per_second.normalized_per_processor": 51.08, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 47.619, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "86fbab953aeb48b0", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7517471, "90.00 percentile latency (ns)": 8377548, "90th percentile latency (ns)": 8377548, "95.00 percentile latency (ns)": 10177424, "97.00 percentile latency (ns)": 16899539, "99.00 percentile latency (ns)": 43189895, "99.90 percentile latency (ns)": 164456911, "Max latency (ns)": 194712795, "Mean latency (ns)": 8889307, "Min duration satisfied": "Yes", "Min latency (ns)": 6826489, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 112.08, "QPS w/o loadgen overhead": 112.49, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.12, "characteristics.good": 29560, "characteristics.samples_per_second": 112.49, "characteristics.samples_per_second.normalized_per_core": 18.74833333333333, "characteristics.samples_per_second.normalized_per_processor": 112.49, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "3434b0bba265e6a8", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7930698, "90.00 percentile latency (ns)": 9282319, "90th percentile latency (ns)": 9282319, "95.00 percentile latency (ns)": 11159779, "97.00 percentile latency (ns)": 21056606, "99.00 percentile latency (ns)": 63261785, "99.90 percentile latency (ns)": 103318463, "Max latency (ns)": 117994734, "Mean latency (ns)": 9471033, "Min duration satisfied": "Yes", "Min latency (ns)": 7117856, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 105.2, "QPS w/o loadgen overhead": 105.59, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.46, "characteristics.good": 29230, "characteristics.samples_per_second": 105.59, "characteristics.samples_per_second.normalized_per_core": 17.598333333333333, "characteristics.samples_per_second.normalized_per_processor": 105.59, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "17ed48379f2db8f6", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 28343272, "90.00 percentile latency (ns)": 54925460, "90th percentile latency (ns)": 54925460, "95.00 percentile latency (ns)": 105130884, "97.00 percentile latency (ns)": 156958972, "99.00 percentile latency (ns)": 185274709, "99.90 percentile latency (ns)": 199068777, "Max latency (ns)": 206972320, "Mean latency (ns)": 39074021, "Min duration satisfied": "Yes", "Min latency (ns)": 27387188, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 25.56, "QPS w/o loadgen overhead": 25.59, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.91, "characteristics.good": 36455, "characteristics.samples_per_second": 25.59, "characteristics.samples_per_second.normalized_per_core": 4.265, "characteristics.samples_per_second.normalized_per_processor": 25.59, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 27.027, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "28e56474c1e9ab98", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 26494628, "90.00 percentile latency (ns)": 49034181, "90th percentile latency (ns)": 49034181, "95.00 percentile latency (ns)": 92485159, "97.00 percentile latency (ns)": 155239706, "99.00 percentile latency (ns)": 194591374, "99.90 percentile latency (ns)": 208654179, "Max latency (ns)": 222012130, "Mean latency (ns)": 36554990, "Min duration satisfied": "Yes", "Min latency (ns)": 25653508, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 27.32, "QPS w/o loadgen overhead": 27.36, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.542, "characteristics.good": 35271, "characteristics.samples_per_second": 27.36, "characteristics.samples_per_second.normalized_per_core": 4.56, "characteristics.samples_per_second.normalized_per_processor": 27.36, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 25.641, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "7af899398c1cf162", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9210561, "90.00 percentile latency (ns)": 10080606, "90th percentile latency (ns)": 10080606, "95.00 percentile latency (ns)": 12425320, "97.00 percentile latency (ns)": 22780588, "99.00 percentile latency (ns)": 65421873, "99.90 percentile latency (ns)": 174927510, "Max latency (ns)": 205106634, "Mean latency (ns)": 10804334, "Min duration satisfied": "Yes", "Min latency (ns)": 7813474, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 92.25, "QPS w/o loadgen overhead": 92.56, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.594, "characteristics.good": 28297, "characteristics.samples_per_second": 92.56, "characteristics.samples_per_second.normalized_per_core": 15.426666666666668, "characteristics.samples_per_second.normalized_per_processor": 92.56, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "42c441882ce30222", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 16382042, "90.00 percentile latency (ns)": 20234668, "90th percentile latency (ns)": 20234668, "95.00 percentile latency (ns)": 35495210, "97.00 percentile latency (ns)": 50017293, "99.00 percentile latency (ns)": 129623381, "99.90 percentile latency (ns)": 191492889, "Max latency (ns)": 209920090, "Mean latency (ns)": 19882047, "Min duration satisfied": "Yes", "Min latency (ns)": 14825709, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 50.19, "QPS w/o loadgen overhead": 50.3, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.82, "characteristics.good": 33910, "characteristics.samples_per_second": 50.3, "characteristics.samples_per_second.normalized_per_core": 8.383333333333333, "characteristics.samples_per_second.normalized_per_processor": 50.3, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "c25ec30471eb967a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4868214, "90.00 percentile latency (ns)": 35494840, "90th percentile latency (ns)": 35494840, "95.00 percentile latency (ns)": 64486175, "97.00 percentile latency (ns)": 76170783, "99.00 percentile latency (ns)": 85462701, "99.90 percentile latency (ns)": 96201139, "Max latency (ns)": 108830150, "Mean latency (ns)": 12620822, "Min duration satisfied": "Yes", "Min latency (ns)": 4572172, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 78.93, "QPS w/o loadgen overhead": 79.23, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 44.116, "characteristics.good": 22058, "characteristics.samples_per_second": 79.23, "characteristics.samples_per_second.normalized_per_core": 13.205, "characteristics.samples_per_second.normalized_per_processor": 79.23, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "2b26b4b3a87420fa", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3958506, "90.00 percentile latency (ns)": 4203799, "90th percentile latency (ns)": 4203799, "95.00 percentile latency (ns)": 5117884, "97.00 percentile latency (ns)": 5860760, "99.00 percentile latency (ns)": 20880161, "99.90 percentile latency (ns)": 82237296, "Max latency (ns)": 105842499, "Mean latency (ns)": 4395911, "Min duration satisfied": "Yes", "Min latency (ns)": 3143880, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 225.98, "QPS w/o loadgen overhead": 227.48, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 45.494, "characteristics.good": 22747, "characteristics.samples_per_second": 227.48, "characteristics.samples_per_second.normalized_per_core": 37.913333333333334, "characteristics.samples_per_second.normalized_per_processor": 227.48, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "e8ecf4e159f82567", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 47220940, "90.00 percentile latency (ns)": 99206589, "90th percentile latency (ns)": 99206589, "95.00 percentile latency (ns)": 130194464, "97.00 percentile latency (ns)": 136852304, "99.00 percentile latency (ns)": 146423124, "99.90 percentile latency (ns)": 156444489, "Max latency (ns)": 168851599, "Mean latency (ns)": 59568211, "Min duration satisfied": "Yes", "Min latency (ns)": 44499111, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.78, "QPS w/o loadgen overhead": 16.79, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.626, "characteristics.good": 35313, "characteristics.samples_per_second": 16.79, "characteristics.samples_per_second.normalized_per_core": 2.7983333333333333, "characteristics.samples_per_second.normalized_per_processor": 16.79, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.625, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "28981fb69c07bd09", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11149264, "90.00 percentile latency (ns)": 13011557, "90th percentile latency (ns)": 13011557, "95.00 percentile latency (ns)": 21248080, "97.00 percentile latency (ns)": 30889896, "99.00 percentile latency (ns)": 76446098, "99.90 percentile latency (ns)": 178062297, "Max latency (ns)": 198543587, "Mean latency (ns)": 13432322, "Min duration satisfied": "Yes", "Min latency (ns)": 10083519, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 74.25, "QPS w/o loadgen overhead": 74.45, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.454, "characteristics.good": 31727, "characteristics.samples_per_second": 74.45, "characteristics.samples_per_second.normalized_per_core": 12.408333333333333, "characteristics.samples_per_second.normalized_per_processor": 74.45, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "4812a11a7a390ef0", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 22883863, "90.00 percentile latency (ns)": 31358527, "90th percentile latency (ns)": 31358527, "95.00 percentile latency (ns)": 46005227, "97.00 percentile latency (ns)": 62535262, "99.00 percentile latency (ns)": 112243495, "99.90 percentile latency (ns)": 125623398, "Max latency (ns)": 135226530, "Mean latency (ns)": 26546330, "Min duration satisfied": "Yes", "Min latency (ns)": 21253238, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 37.61, "QPS w/o loadgen overhead": 37.67, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.04, "characteristics.good": 35020, "characteristics.samples_per_second": 37.67, "characteristics.samples_per_second.normalized_per_core": 6.278333333333333, "characteristics.samples_per_second.normalized_per_processor": 37.67, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 37.037, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "db8ee4b1246f3a32", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 82084321, "90.00 percentile latency (ns)": 218946291, "90th percentile latency (ns)": 218946291, "95.00 percentile latency (ns)": 240552618, "97.00 percentile latency (ns)": 253173361, "99.00 percentile latency (ns)": 264615158, "99.90 percentile latency (ns)": 280148174, "Max latency (ns)": 299587688, "Mean latency (ns)": 115944873, "Min duration satisfied": "Yes", "Min latency (ns)": 71941036, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 8.62, "QPS w/o loadgen overhead": 8.62, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.124, "characteristics.good": 37062, "characteristics.samples_per_second": 8.62, "characteristics.samples_per_second.normalized_per_core": 1.4366666666666665, "characteristics.samples_per_second.normalized_per_processor": 8.62, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite2-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 9.25926, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "4ea9329d028ff7c7", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6887138, "90.00 percentile latency (ns)": 7588599, "90th percentile latency (ns)": 7588599, "95.00 percentile latency (ns)": 9032935, "97.00 percentile latency (ns)": 13319692, "99.00 percentile latency (ns)": 45967515, "99.90 percentile latency (ns)": 99540194, "Max latency (ns)": 111103213, "Mean latency (ns)": 8016714, "Min duration satisfied": "Yes", "Min latency (ns)": 6044803, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 124.25, "QPS w/o loadgen overhead": 124.74, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.444, "characteristics.good": 29722, "characteristics.samples_per_second": 124.74, "characteristics.samples_per_second.normalized_per_core": 20.79, "characteristics.samples_per_second.normalized_per_processor": 124.74, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "55f8ce294b4ebd55", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8911875, "90.00 percentile latency (ns)": 9939708, "90th percentile latency (ns)": 9939708, "95.00 percentile latency (ns)": 12300745, "97.00 percentile latency (ns)": 22090457, "99.00 percentile latency (ns)": 63948113, "99.90 percentile latency (ns)": 103096118, "Max latency (ns)": 117359647, "Mean latency (ns)": 10254123, "Min duration satisfied": "Yes", "Min latency (ns)": 7831250, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 97.22, "QPS w/o loadgen overhead": 97.52, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.456, "characteristics.good": 30728, "characteristics.samples_per_second": 97.52, "characteristics.samples_per_second.normalized_per_core": 16.253333333333334, "characteristics.samples_per_second.normalized_per_processor": 97.52, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "fdcc0fb246d19aeb", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10464424, "90.00 percentile latency (ns)": 13550840, "90th percentile latency (ns)": 13550840, "95.00 percentile latency (ns)": 24924679, "97.00 percentile latency (ns)": 39612849, "99.00 percentile latency (ns)": 144004531, "99.90 percentile latency (ns)": 182001982, "Max latency (ns)": 197312885, "Mean latency (ns)": 14159493, "Min duration satisfied": "Yes", "Min latency (ns)": 10007963, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 70.41, "QPS w/o loadgen overhead": 70.62, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.528, "characteristics.good": 29264, "characteristics.samples_per_second": 70.62, "characteristics.samples_per_second.normalized_per_core": 11.770000000000001, "characteristics.samples_per_second.normalized_per_processor": 70.62, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 62.5, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "c588145e1b9dcd83", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 26512340, "90.00 percentile latency (ns)": 169755088, "90th percentile latency (ns)": 169755088, "95.00 percentile latency (ns)": 183360654, "97.00 percentile latency (ns)": 188454435, "99.00 percentile latency (ns)": 195369950, "99.90 percentile latency (ns)": 205849329, "Max latency (ns)": 216695538, "Mean latency (ns)": 58189980, "Min duration satisfied": "Yes", "Min latency (ns)": 19906298, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.17, "QPS w/o loadgen overhead": 17.19, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.036, "characteristics.good": 33018, "characteristics.samples_per_second": 17.19, "characteristics.samples_per_second.normalized_per_core": 2.865, "characteristics.samples_per_second.normalized_per_processor": 17.19, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.1515, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "95ac7186795038b1", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 106646431, "90.00 percentile latency (ns)": 188719818, "90th percentile latency (ns)": 188719818, "95.00 percentile latency (ns)": 251657516, "97.00 percentile latency (ns)": 269446840, "99.00 percentile latency (ns)": 285280972, "99.90 percentile latency (ns)": 298914223, "Max latency (ns)": 318689039, "Mean latency (ns)": 122777928, "Min duration satisfied": "Yes", "Min latency (ns)": 92435266, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 8.14, "QPS w/o loadgen overhead": 8.14, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.642, "characteristics.good": 38321, "characteristics.samples_per_second": 8.14, "characteristics.samples_per_second.normalized_per_core": 1.3566666666666667, "characteristics.samples_per_second.normalized_per_processor": 8.14, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite1-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 7.69231, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "b5084b66b44e19bd", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 21975023, "90.00 percentile latency (ns)": 29024868, "90th percentile latency (ns)": 29024868, "95.00 percentile latency (ns)": 44865639, "97.00 percentile latency (ns)": 61585670, "99.00 percentile latency (ns)": 148401649, "99.90 percentile latency (ns)": 196581733, "Max latency (ns)": 219009179, "Mean latency (ns)": 25987340, "Min duration satisfied": "Yes", "Min latency (ns)": 19767113, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 38.42, "QPS w/o loadgen overhead": 38.48, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.604, "characteristics.good": 34802, "characteristics.samples_per_second": 38.48, "characteristics.samples_per_second.normalized_per_core": 6.413333333333333, "characteristics.samples_per_second.normalized_per_processor": 38.48, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 35.7143, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "aae7e543b6aa65b2", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 42093860, "90.00 percentile latency (ns)": 103966946, "90th percentile latency (ns)": 103966946, "95.00 percentile latency (ns)": 126031913, "97.00 percentile latency (ns)": 130761700, "99.00 percentile latency (ns)": 137410859, "99.90 percentile latency (ns)": 147584181, "Max latency (ns)": 153754085, "Mean latency (ns)": 55490086, "Min duration satisfied": "Yes", "Min latency (ns)": 39601280, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 18.01, "QPS w/o loadgen overhead": 18.02, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.262, "characteristics.good": 34631, "characteristics.samples_per_second": 18.02, "characteristics.samples_per_second.normalized_per_core": 3.0033333333333334, "characteristics.samples_per_second.normalized_per_processor": 18.02, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 16.129, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "2aede5494da067c7", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 21574324, "90.00 percentile latency (ns)": 36915725, "90th percentile latency (ns)": 36915725, "95.00 percentile latency (ns)": 57276216, "97.00 percentile latency (ns)": 108585630, "99.00 percentile latency (ns)": 179276399, "99.90 percentile latency (ns)": 198989599, "Max latency (ns)": 215087503, "Mean latency (ns)": 28822997, "Min duration satisfied": "Yes", "Min latency (ns)": 20932366, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 34.64, "QPS w/o loadgen overhead": 34.69, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.734, "characteristics.good": 34367, "characteristics.samples_per_second": 34.69, "characteristics.samples_per_second.normalized_per_core": 5.781666666666666, "characteristics.samples_per_second.normalized_per_processor": 34.69, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 27.7778, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "fa4ce940ffac2565", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 22166363, "90.00 percentile latency (ns)": 118090833, "90th percentile latency (ns)": 118090833, "95.00 percentile latency (ns)": 125432297, "97.00 percentile latency (ns)": 128582941, "99.00 percentile latency (ns)": 133061336, "99.90 percentile latency (ns)": 141411892, "Max latency (ns)": 151787647, "Mean latency (ns)": 43672034, "Min duration satisfied": "Yes", "Min latency (ns)": 15316868, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 22.88, "QPS w/o loadgen overhead": 22.9, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.33, "characteristics.good": 31665, "characteristics.samples_per_second": 22.9, "characteristics.samples_per_second.normalized_per_core": 3.8166666666666664, "characteristics.samples_per_second.normalized_per_processor": 22.9, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.1515, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "5856cf494bb248ab", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10253257, "90.00 percentile latency (ns)": 78455495, "90th percentile latency (ns)": 78455495, "95.00 percentile latency (ns)": 93172117, "97.00 percentile latency (ns)": 96517047, "99.00 percentile latency (ns)": 101052684, "99.90 percentile latency (ns)": 108020001, "Max latency (ns)": 115166725, "Mean latency (ns)": 23420636, "Min duration satisfied": "Yes", "Min latency (ns)": 8704214, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 42.58, "QPS w/o loadgen overhead": 42.7, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 57.918, "characteristics.good": 28959, "characteristics.samples_per_second": 42.7, "characteristics.samples_per_second.normalized_per_core": 7.116666666666667, "characteristics.samples_per_second.normalized_per_processor": 42.7, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "43f5a38490274b2b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 67113534, "90.00 percentile latency (ns)": 112243387, "90th percentile latency (ns)": 112243387, "95.00 percentile latency (ns)": 140090271, "97.00 percentile latency (ns)": 147007330, "99.00 percentile latency (ns)": 154845317, "99.90 percentile latency (ns)": 164764637, "Max latency (ns)": 173258843, "Mean latency (ns)": 75139987, "Min duration satisfied": "Yes", "Min latency (ns)": 58561334, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 13.3, "QPS w/o loadgen overhead": 13.31, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.676, "characteristics.good": 35838, "characteristics.samples_per_second": 13.31, "characteristics.samples_per_second.normalized_per_core": 2.2183333333333333, "characteristics.samples_per_second.normalized_per_processor": 13.31, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.987, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "4e41083b41e19792", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3665669, "90.00 percentile latency (ns)": 3910669, "90th percentile latency (ns)": 3910669, "95.00 percentile latency (ns)": 4677169, "97.00 percentile latency (ns)": 5617795, "99.00 percentile latency (ns)": 18406510, "99.90 percentile latency (ns)": 106049478, "Max latency (ns)": 168409023, "Mean latency (ns)": 4138556, "Min duration satisfied": "Yes", "Min latency (ns)": 3153794, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 239.27, "QPS w/o loadgen overhead": 241.63, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 43.468, "characteristics.good": 21734, "characteristics.samples_per_second": 241.63, "characteristics.samples_per_second.normalized_per_core": 40.27166666666667, "characteristics.samples_per_second.normalized_per_processor": 241.63, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "9376c6ecae46fea1", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 493166225, "90.00 percentile latency (ns)": 545062465, "90th percentile latency (ns)": 545062465, "95.00 percentile latency (ns)": 558331316, "97.00 percentile latency (ns)": 563402405, "99.00 percentile latency (ns)": 578899309, "99.90 percentile latency (ns)": 603919584, "Max latency (ns)": 606813127, "Mean latency (ns)": 486407759, "Min duration satisfied": "Yes", "Min latency (ns)": 317799355, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.06, "QPS w/o loadgen overhead": 2.06, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 80.676, "characteristics.good": 40338, "characteristics.samples_per_second": 2.06, "characteristics.samples_per_second.normalized_per_core": 0.3433333333333333, "characteristics.samples_per_second.normalized_per_processor": 2.06, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite4-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.04082, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "baefa2034905b557", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20320465, "90.00 percentile latency (ns)": 37608561, "90th percentile latency (ns)": 37608561, "95.00 percentile latency (ns)": 62329245, "97.00 percentile latency (ns)": 113245349, "99.00 percentile latency (ns)": 171833717, "99.90 percentile latency (ns)": 186307213, "Max latency (ns)": 204630543, "Mean latency (ns)": 27927407, "Min duration satisfied": "Yes", "Min latency (ns)": 19294991, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 35.75, "QPS w/o loadgen overhead": 35.81, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.43, "characteristics.good": 35215, "characteristics.samples_per_second": 35.81, "characteristics.samples_per_second.normalized_per_core": 5.968333333333334, "characteristics.samples_per_second.normalized_per_processor": 35.81, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 31.25, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "461f124961b743e2", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 17615222, "90.00 percentile latency (ns)": 22136642, "90th percentile latency (ns)": 22136642, "95.00 percentile latency (ns)": 38915758, "97.00 percentile latency (ns)": 56554938, "99.00 percentile latency (ns)": 152838474, "99.90 percentile latency (ns)": 195292286, "Max latency (ns)": 207265853, "Mean latency (ns)": 21510970, "Min duration satisfied": "Yes", "Min latency (ns)": 15719395, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 46.39, "QPS w/o loadgen overhead": 46.49, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.474, "characteristics.good": 33737, "characteristics.samples_per_second": 46.49, "characteristics.samples_per_second.normalized_per_core": 7.748333333333334, "characteristics.samples_per_second.normalized_per_processor": 46.49, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "b45acc2640550b64", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 14653339, "90.00 percentile latency (ns)": 18213715, "90th percentile latency (ns)": 18213715, "95.00 percentile latency (ns)": 32555846, "97.00 percentile latency (ns)": 44771434, "99.00 percentile latency (ns)": 99529581, "99.90 percentile latency (ns)": 116789545, "Max latency (ns)": 137581845, "Mean latency (ns)": 17458367, "Min duration satisfied": "Yes", "Min latency (ns)": 13464214, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 57.15, "QPS w/o loadgen overhead": 57.28, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.24, "characteristics.good": 33620, "characteristics.samples_per_second": 57.28, "characteristics.samples_per_second.normalized_per_core": 9.546666666666667, "characteristics.samples_per_second.normalized_per_processor": 57.28, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 55.5556, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "85c118c97f340059", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 29050615, "90.00 percentile latency (ns)": 111500635, "90th percentile latency (ns)": 111500635, "95.00 percentile latency (ns)": 116039769, "97.00 percentile latency (ns)": 118585473, "99.00 percentile latency (ns)": 124206417, "99.90 percentile latency (ns)": 132168918, "Max latency (ns)": 141832409, "Mean latency (ns)": 48846160, "Min duration satisfied": "Yes", "Min latency (ns)": 22974915, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.45, "QPS w/o loadgen overhead": 20.47, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.348, "characteristics.good": 32174, "characteristics.samples_per_second": 20.47, "characteristics.samples_per_second.normalized_per_core": 3.4116666666666666, "characteristics.samples_per_second.normalized_per_processor": 20.47, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.6582, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "4bc5ba9fba46fb0e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 24845427, "90.00 percentile latency (ns)": 106397406, "90th percentile latency (ns)": 106397406, "95.00 percentile latency (ns)": 110835831, "97.00 percentile latency (ns)": 113210040, "99.00 percentile latency (ns)": 118383452, "99.90 percentile latency (ns)": 126807003, "Max latency (ns)": 133033629, "Mean latency (ns)": 46232417, "Min duration satisfied": "Yes", "Min latency (ns)": 16990535, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 21.6, "QPS w/o loadgen overhead": 21.63, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.246, "characteristics.good": 31623, "characteristics.samples_per_second": 21.63, "characteristics.samples_per_second.normalized_per_core": 3.605, "characteristics.samples_per_second.normalized_per_processor": 21.63, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 16.9492, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "3db57edc65640d09", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20302919, "90.00 percentile latency (ns)": 149681596, "90th percentile latency (ns)": 149681596, "95.00 percentile latency (ns)": 171478140, "97.00 percentile latency (ns)": 177026518, "99.00 percentile latency (ns)": 185637602, "99.90 percentile latency (ns)": 199486606, "Max latency (ns)": 209828482, "Mean latency (ns)": 48166814, "Min duration satisfied": "Yes", "Min latency (ns)": 14214377, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.74, "QPS w/o loadgen overhead": 20.76, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.668, "characteristics.good": 30834, "characteristics.samples_per_second": 20.76, "characteristics.samples_per_second.normalized_per_core": 3.4600000000000004, "characteristics.samples_per_second.normalized_per_processor": 20.76, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.9254, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "772fa3077f406f9d", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 27680589, "90.00 percentile latency (ns)": 111265732, "90th percentile latency (ns)": 111265732, "95.00 percentile latency (ns)": 116376725, "97.00 percentile latency (ns)": 118541431, "99.00 percentile latency (ns)": 122734634, "99.90 percentile latency (ns)": 131910456, "Max latency (ns)": 170061947, "Mean latency (ns)": 47738250, "Min duration satisfied": "Yes", "Min latency (ns)": 23015678, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.92, "QPS w/o loadgen overhead": 20.95, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.384, "characteristics.good": 33192, "characteristics.samples_per_second": 20.95, "characteristics.samples_per_second.normalized_per_core": 3.4916666666666667, "characteristics.samples_per_second.normalized_per_processor": 20.95, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 21.2766, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "655992d2857cfaa0", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4536301, "90.00 percentile latency (ns)": 4827087, "90th percentile latency (ns)": 4827087, "95.00 percentile latency (ns)": 5834796, "97.00 percentile latency (ns)": 7165379, "99.00 percentile latency (ns)": 25497262, "99.90 percentile latency (ns)": 129616094, "Max latency (ns)": 177859017, "Mean latency (ns)": 5248003, "Min duration satisfied": "Yes", "Min latency (ns)": 4066717, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 189.23, "QPS w/o loadgen overhead": 190.55, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.11, "characteristics.good": 24555, "characteristics.samples_per_second": 190.55, "characteristics.samples_per_second.normalized_per_core": 31.758333333333336, "characteristics.samples_per_second.normalized_per_processor": 190.55, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "e6fea14d5287afdf", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9345883, "90.00 percentile latency (ns)": 10262592, "90th percentile latency (ns)": 10262592, "95.00 percentile latency (ns)": 13389261, "97.00 percentile latency (ns)": 23671271, "99.00 percentile latency (ns)": 52784919, "99.90 percentile latency (ns)": 169222984, "Max latency (ns)": 190353819, "Mean latency (ns)": 11014115, "Min duration satisfied": "Yes", "Min latency (ns)": 8397382, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 90.52, "QPS w/o loadgen overhead": 90.79, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 62.07, "characteristics.good": 31035, "characteristics.samples_per_second": 90.79, "characteristics.samples_per_second.normalized_per_core": 15.131666666666668, "characteristics.samples_per_second.normalized_per_processor": 90.79, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "48e4b58d036b1d2f", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 22491892, "90.00 percentile latency (ns)": 159527287, "90th percentile latency (ns)": 159527287, "95.00 percentile latency (ns)": 175158210, "97.00 percentile latency (ns)": 180113523, "99.00 percentile latency (ns)": 186679424, "99.90 percentile latency (ns)": 196358300, "Max latency (ns)": 207891714, "Mean latency (ns)": 52277473, "Min duration satisfied": "Yes", "Min latency (ns)": 14357010, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 19.11, "QPS w/o loadgen overhead": 19.13, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.224, "characteristics.good": 30612, "characteristics.samples_per_second": 19.13, "characteristics.samples_per_second.normalized_per_core": 3.188333333333333, "characteristics.samples_per_second.normalized_per_processor": 19.13, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.0845, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "2aaa031ac4b8b920", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2730588, "90.00 percentile latency (ns)": 2926880, "90th percentile latency (ns)": 2926880, "95.00 percentile latency (ns)": 3304880, "97.00 percentile latency (ns)": 4201173, "99.00 percentile latency (ns)": 10234308, "99.90 percentile latency (ns)": 76645992, "Max latency (ns)": 97319692, "Mean latency (ns)": 3024682, "Min duration satisfied": "Yes", "Min latency (ns)": 2192462, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 326.77, "QPS w/o loadgen overhead": 330.61, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 42.384, "characteristics.good": 21192, "characteristics.samples_per_second": 330.61, "characteristics.samples_per_second.normalized_per_core": 55.10166666666667, "characteristics.samples_per_second.normalized_per_processor": 330.61, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "22fb6db0104e1b9f", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 90824085, "90.00 percentile latency (ns)": 214439020, "90th percentile latency (ns)": 214439020, "95.00 percentile latency (ns)": 231803674, "97.00 percentile latency (ns)": 237869912, "99.00 percentile latency (ns)": 246492457, "99.90 percentile latency (ns)": 258797746, "Max latency (ns)": 265246279, "Mean latency (ns)": 115129795, "Min duration satisfied": "Yes", "Min latency (ns)": 75316119, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 8.68, "QPS w/o loadgen overhead": 8.69, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.012, "characteristics.good": 37506, "characteristics.samples_per_second": 8.69, "characteristics.samples_per_second.normalized_per_core": 1.4483333333333333, "characteristics.samples_per_second.normalized_per_processor": 8.69, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.3-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.3-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.3_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 7.46269, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "1db248638dc1dd4b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 12214715, "90.00 percentile latency (ns)": 20767838, "90th percentile latency (ns)": 20767838, "95.00 percentile latency (ns)": 37794467, "97.00 percentile latency (ns)": 75555898, "99.00 percentile latency (ns)": 120644553, "99.90 percentile latency (ns)": 133118886, "Max latency (ns)": 145581929, "Mean latency (ns)": 17070558, "Min duration satisfied": "Yes", "Min latency (ns)": 11446458, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 58.37, "QPS w/o loadgen overhead": 58.58, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.486, "characteristics.good": 32243, "characteristics.samples_per_second": 58.58, "characteristics.samples_per_second.normalized_per_core": 9.763333333333334, "characteristics.samples_per_second.normalized_per_processor": 58.58, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 41.6667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "de6294a05a12d46d", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 235354940, "90.00 percentile latency (ns)": 293672880, "90th percentile latency (ns)": 293672880, "95.00 percentile latency (ns)": 303602443, "97.00 percentile latency (ns)": 309512456, "99.00 percentile latency (ns)": 317951459, "99.90 percentile latency (ns)": 341667413, "Max latency (ns)": 448119929, "Mean latency (ns)": 226369438, "Min duration satisfied": "Yes", "Min latency (ns)": 123688929, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 4.42, "QPS w/o loadgen overhead": 4.42, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 77.542, "characteristics.good": 38771, "characteristics.samples_per_second": 4.42, "characteristics.samples_per_second.normalized_per_core": 0.7366666666666667, "characteristics.samples_per_second.normalized_per_processor": 4.42, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite2-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 4.1841, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "3e3b0e38f4eb9800", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4758237, "90.00 percentile latency (ns)": 5070901, "90th percentile latency (ns)": 5070901, "95.00 percentile latency (ns)": 6425106, "97.00 percentile latency (ns)": 7324606, "99.00 percentile latency (ns)": 30748539, "99.90 percentile latency (ns)": 92909809, "Max latency (ns)": 110418255, "Mean latency (ns)": 5556298, "Min duration satisfied": "Yes", "Min latency (ns)": 4188612, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 178.91, "QPS w/o loadgen overhead": 179.98, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.75, "characteristics.good": 28375, "characteristics.samples_per_second": 179.98, "characteristics.samples_per_second.normalized_per_core": 29.996666666666666, "characteristics.samples_per_second.normalized_per_processor": 179.98, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "482b9551974f1491", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6363308, "90.00 percentile latency (ns)": 6912225, "90th percentile latency (ns)": 6912225, "95.00 percentile latency (ns)": 8768984, "97.00 percentile latency (ns)": 12195198, "99.00 percentile latency (ns)": 35451960, "99.90 percentile latency (ns)": 160676373, "Max latency (ns)": 190830690, "Mean latency (ns)": 7541181, "Min duration satisfied": "Yes", "Min latency (ns)": 5678180, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 131.99, "QPS w/o loadgen overhead": 132.61, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 57.002, "characteristics.good": 28501, "characteristics.samples_per_second": 132.61, "characteristics.samples_per_second.normalized_per_core": 22.10166666666667, "characteristics.samples_per_second.normalized_per_processor": 132.61, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "1dac76ebd4452a52", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 97122266, "90.00 percentile latency (ns)": 224166652, "90th percentile latency (ns)": 224166652, "95.00 percentile latency (ns)": 239890229, "97.00 percentile latency (ns)": 245796986, "99.00 percentile latency (ns)": 254903706, "99.90 percentile latency (ns)": 269859817, "Max latency (ns)": 302025508, "Mean latency (ns)": 122297193, "Min duration satisfied": "Yes", "Min latency (ns)": 80872906, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 8.17, "QPS w/o loadgen overhead": 8.18, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.69, "characteristics.good": 37845, "characteristics.samples_per_second": 8.18, "characteristics.samples_per_second.normalized_per_core": 1.3633333333333333, "characteristics.samples_per_second.normalized_per_processor": 8.18, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.4-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.4-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.4_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 7.8125, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "9e7406d814cd0a0c", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 12780032, "90.00 percentile latency (ns)": 87989630, "90th percentile latency (ns)": 87989630, "95.00 percentile latency (ns)": 128423714, "97.00 percentile latency (ns)": 146326964, "99.00 percentile latency (ns)": 161816515, "99.90 percentile latency (ns)": 175251622, "Max latency (ns)": 221374888, "Mean latency (ns)": 32478406, "Min duration satisfied": "Yes", "Min latency (ns)": 7508101, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 30.74, "QPS w/o loadgen overhead": 30.79, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 52.446, "characteristics.good": 26223, "characteristics.samples_per_second": 30.79, "characteristics.samples_per_second.normalized_per_core": 5.131666666666667, "characteristics.samples_per_second.normalized_per_processor": 30.79, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.2558, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "14eddc4e9cbea76e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 40724219, "90.00 percentile latency (ns)": 142885348, "90th percentile latency (ns)": 142885348, "95.00 percentile latency (ns)": 178986945, "97.00 percentile latency (ns)": 184916233, "99.00 percentile latency (ns)": 192241144, "99.90 percentile latency (ns)": 203334135, "Max latency (ns)": 211794462, "Mean latency (ns)": 62287004, "Min duration satisfied": "Yes", "Min latency (ns)": 36586638, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.05, "QPS w/o loadgen overhead": 16.05, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.724, "characteristics.good": 36862, "characteristics.samples_per_second": 16.05, "characteristics.samples_per_second.normalized_per_core": 2.6750000000000003, "characteristics.samples_per_second.normalized_per_processor": 16.05, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.0482, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "1cb8e155f782a4ea", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 32019478, "90.00 percentile latency (ns)": 112981237, "90th percentile latency (ns)": 112981237, "95.00 percentile latency (ns)": 117429446, "97.00 percentile latency (ns)": 119822574, "99.00 percentile latency (ns)": 124209659, "99.90 percentile latency (ns)": 132818665, "Max latency (ns)": 191395578, "Mean latency (ns)": 52205445, "Min duration satisfied": "Yes", "Min latency (ns)": 23986389, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 19.13, "QPS w/o loadgen overhead": 19.16, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.604, "characteristics.good": 33302, "characteristics.samples_per_second": 19.16, "characteristics.samples_per_second.normalized_per_core": 3.1933333333333334, "characteristics.samples_per_second.normalized_per_processor": 19.16, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.873, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "989c8915064e6612", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 36871071, "90.00 percentile latency (ns)": 109830506, "90th percentile latency (ns)": 109830506, "95.00 percentile latency (ns)": 123202720, "97.00 percentile latency (ns)": 126797048, "99.00 percentile latency (ns)": 133317612, "99.90 percentile latency (ns)": 142310702, "Max latency (ns)": 190740268, "Mean latency (ns)": 52429974, "Min duration satisfied": "Yes", "Min latency (ns)": 34244324, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 19.06, "QPS w/o loadgen overhead": 19.07, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.106, "characteristics.good": 34553, "characteristics.samples_per_second": 19.07, "characteristics.samples_per_second.normalized_per_core": 3.1783333333333332, "characteristics.samples_per_second.normalized_per_processor": 19.07, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 16.9492, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "b1fb7f7a84661883", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 19473532, "90.00 percentile latency (ns)": 144222804, "90th percentile latency (ns)": 144222804, "95.00 percentile latency (ns)": 165272168, "97.00 percentile latency (ns)": 170697082, "99.00 percentile latency (ns)": 178262160, "99.90 percentile latency (ns)": 191207190, "Max latency (ns)": 220197485, "Mean latency (ns)": 46844565, "Min duration satisfied": "Yes", "Min latency (ns)": 11443613, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 21.33, "QPS w/o loadgen overhead": 21.35, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.822, "characteristics.good": 29911, "characteristics.samples_per_second": 21.35, "characteristics.samples_per_second.normalized_per_core": 3.5583333333333336, "characteristics.samples_per_second.normalized_per_processor": 21.35, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.7059, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "495e804ea1e81d70", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20471162, "90.00 percentile latency (ns)": 90984416, "90th percentile latency (ns)": 90984416, "95.00 percentile latency (ns)": 126942960, "97.00 percentile latency (ns)": 131793943, "99.00 percentile latency (ns)": 137545287, "99.90 percentile latency (ns)": 146154368, "Max latency (ns)": 153606213, "Mean latency (ns)": 34993742, "Min duration satisfied": "Yes", "Min latency (ns)": 19234206, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 28.55, "QPS w/o loadgen overhead": 28.58, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.336, "characteristics.good": 34168, "characteristics.samples_per_second": 28.58, "characteristics.samples_per_second.normalized_per_core": 4.763333333333333, "characteristics.samples_per_second.normalized_per_processor": 28.58, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 19.6078, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "b77b44209b747a85", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 30922372, "90.00 percentile latency (ns)": 100687698, "90th percentile latency (ns)": 100687698, "95.00 percentile latency (ns)": 166090576, "97.00 percentile latency (ns)": 183803355, "99.00 percentile latency (ns)": 195721928, "99.90 percentile latency (ns)": 208600006, "Max latency (ns)": 222734699, "Mean latency (ns)": 49149669, "Min duration satisfied": "Yes", "Min latency (ns)": 29720316, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.33, "QPS w/o loadgen overhead": 20.35, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.834, "characteristics.good": 34417, "characteristics.samples_per_second": 20.35, "characteristics.samples_per_second.normalized_per_core": 3.391666666666667, "characteristics.samples_per_second.normalized_per_processor": 20.35, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 13.8889, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "05be63aba8357e2d", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 37430736, "90.00 percentile latency (ns)": 76605925, "90th percentile latency (ns)": 76605925, "95.00 percentile latency (ns)": 139227907, "97.00 percentile latency (ns)": 168307146, "99.00 percentile latency (ns)": 186914303, "99.90 percentile latency (ns)": 201449122, "Max latency (ns)": 213005614, "Mean latency (ns)": 49984386, "Min duration satisfied": "Yes", "Min latency (ns)": 34854737, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 19.98, "QPS w/o loadgen overhead": 20.01, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.19, "characteristics.good": 35595, "characteristics.samples_per_second": 20.01, "characteristics.samples_per_second.normalized_per_core": 3.3350000000000004, "characteristics.samples_per_second.normalized_per_processor": 20.01, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite0-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.5185, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "94f0ab59abb3c368", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 328586430, "90.00 percentile latency (ns)": 380736463, "90th percentile latency (ns)": 380736463, "95.00 percentile latency (ns)": 390845357, "97.00 percentile latency (ns)": 396542383, "99.00 percentile latency (ns)": 409008172, "99.90 percentile latency (ns)": 430273817, "Max latency (ns)": 436923714, "Mean latency (ns)": 324454976, "Min duration satisfied": "Yes", "Min latency (ns)": 183151052, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 3.08, "QPS w/o loadgen overhead": 3.08, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 79.268, "characteristics.good": 39634, "characteristics.samples_per_second": 3.08, "characteristics.samples_per_second.normalized_per_core": 0.5133333333333333, "characteristics.samples_per_second.normalized_per_processor": 3.08, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite3-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.0581, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "83a1aa9e93f9c11f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20941283, "90.00 percentile latency (ns)": 120124191, "90th percentile latency (ns)": 120124191, "95.00 percentile latency (ns)": 128274408, "97.00 percentile latency (ns)": 131334281, "99.00 percentile latency (ns)": 136386437, "99.90 percentile latency (ns)": 143850135, "Max latency (ns)": 148640802, "Mean latency (ns)": 42570811, "Min duration satisfied": "Yes", "Min latency (ns)": 16920487, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 23.47, "QPS w/o loadgen overhead": 23.49, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.216, "characteristics.good": 33108, "characteristics.samples_per_second": 23.49, "characteristics.samples_per_second.normalized_per_core": 3.9149999999999996, "characteristics.samples_per_second.normalized_per_processor": 23.49, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.3846, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "a52cc594ca6ac9d6", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 115874378, "90.00 percentile latency (ns)": 269592209, "90th percentile latency (ns)": 269592209, "95.00 percentile latency (ns)": 300265842, "97.00 percentile latency (ns)": 312628754, "99.00 percentile latency (ns)": 325390025, "99.90 percentile latency (ns)": 345693816, "Max latency (ns)": 365999025, "Mean latency (ns)": 150988597, "Min duration satisfied": "Yes", "Min latency (ns)": 104186136, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 6.62, "QPS w/o loadgen overhead": 6.62, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.344, "characteristics.good": 38172, "characteristics.samples_per_second": 6.62, "characteristics.samples_per_second.normalized_per_core": 1.1033333333333333, "characteristics.samples_per_second.normalized_per_processor": 6.62, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite3-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 6.09756, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "6346259cbce85692", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7051358, "90.00 percentile latency (ns)": 7642861, "90th percentile latency (ns)": 7642861, "95.00 percentile latency (ns)": 9089813, "97.00 percentile latency (ns)": 11290455, "99.00 percentile latency (ns)": 36475044, "99.90 percentile latency (ns)": 159079489, "Max latency (ns)": 183746471, "Mean latency (ns)": 8050293, "Min duration satisfied": "Yes", "Min latency (ns)": 6052109, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 123.73, "QPS w/o loadgen overhead": 124.22, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 54.568, "characteristics.good": 27284, "characteristics.samples_per_second": 124.22, "characteristics.samples_per_second.normalized_per_core": 20.703333333333333, "characteristics.samples_per_second.normalized_per_processor": 124.22, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "f6c43eab596e5ab0", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9956947, "90.00 percentile latency (ns)": 22275246, "90th percentile latency (ns)": 22275246, "95.00 percentile latency (ns)": 45031125, "97.00 percentile latency (ns)": 67555777, "99.00 percentile latency (ns)": 89787574, "99.90 percentile latency (ns)": 100707019, "Max latency (ns)": 112317832, "Mean latency (ns)": 14560577, "Min duration satisfied": "Yes", "Min latency (ns)": 8737199, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 68.47, "QPS w/o loadgen overhead": 68.68, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.478, "characteristics.good": 24739, "characteristics.samples_per_second": 68.68, "characteristics.samples_per_second.normalized_per_core": 11.446666666666667, "characteristics.samples_per_second.normalized_per_processor": 68.68, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "676ab4531d4f6391", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 19178790, "90.00 percentile latency (ns)": 38852916, "90th percentile latency (ns)": 38852916, "95.00 percentile latency (ns)": 83031837, "97.00 percentile latency (ns)": 104816710, "99.00 percentile latency (ns)": 113668641, "99.90 percentile latency (ns)": 122058297, "Max latency (ns)": 128536070, "Mean latency (ns)": 26270045, "Min duration satisfied": "Yes", "Min latency (ns)": 18471522, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 37.96, "QPS w/o loadgen overhead": 38.07, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.126, "characteristics.good": 34063, "characteristics.samples_per_second": 38.07, "characteristics.samples_per_second.normalized_per_core": 6.345, "characteristics.samples_per_second.normalized_per_processor": 38.07, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 43.4783, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "e7c48899e3897601", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 47920505, "90.00 percentile latency (ns)": 148144554, "90th percentile latency (ns)": 148144554, "95.00 percentile latency (ns)": 187204362, "97.00 percentile latency (ns)": 195508978, "99.00 percentile latency (ns)": 204716307, "99.90 percentile latency (ns)": 216698547, "Max latency (ns)": 230300413, "Mean latency (ns)": 71103859, "Min duration satisfied": "Yes", "Min latency (ns)": 46066091, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 14.06, "QPS w/o loadgen overhead": 14.06, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.76, "characteristics.good": 37880, "characteristics.samples_per_second": 14.06, "characteristics.samples_per_second.normalized_per_core": 2.3433333333333333, "characteristics.samples_per_second.normalized_per_processor": 14.06, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 13.3333, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "26db415896b4ec29", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 36431825, "90.00 percentile latency (ns)": 175026999, "90th percentile latency (ns)": 175026999, "95.00 percentile latency (ns)": 187267887, "97.00 percentile latency (ns)": 191344378, "99.00 percentile latency (ns)": 197712879, "99.90 percentile latency (ns)": 210563221, "Max latency (ns)": 223541360, "Mean latency (ns)": 65653870, "Min duration satisfied": "Yes", "Min latency (ns)": 25452903, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 15.22, "QPS w/o loadgen overhead": 15.23, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 65.75, "characteristics.good": 32875, "characteristics.samples_per_second": 15.23, "characteristics.samples_per_second.normalized_per_core": 2.5383333333333336, "characteristics.samples_per_second.normalized_per_processor": 15.23, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.2857, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "ba693c247f493c6f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11452855, "90.00 percentile latency (ns)": 13839854, "90th percentile latency (ns)": 13839854, "95.00 percentile latency (ns)": 21852507, "97.00 percentile latency (ns)": 32943390, "99.00 percentile latency (ns)": 82594100, "99.90 percentile latency (ns)": 108376746, "Max latency (ns)": 121321486, "Mean latency (ns)": 13625916, "Min duration satisfied": "Yes", "Min latency (ns)": 10658358, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 73.18, "QPS w/o loadgen overhead": 73.39, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.768, "characteristics.good": 32384, "characteristics.samples_per_second": 73.39, "characteristics.samples_per_second.normalized_per_core": 12.231666666666667, "characteristics.samples_per_second.normalized_per_processor": 73.39, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "05bd4813b48dc85f", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 44642272, "90.00 percentile latency (ns)": 147897162, "90th percentile latency (ns)": 147897162, "95.00 percentile latency (ns)": 192843801, "97.00 percentile latency (ns)": 201030289, "99.00 percentile latency (ns)": 209919503, "99.90 percentile latency (ns)": 221491487, "Max latency (ns)": 229554586, "Mean latency (ns)": 66022015, "Min duration satisfied": "Yes", "Min latency (ns)": 40152632, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 15.14, "QPS w/o loadgen overhead": 15.15, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.122, "characteristics.good": 35061, "characteristics.samples_per_second": 15.15, "characteristics.samples_per_second.normalized_per_core": 2.525, "characteristics.samples_per_second.normalized_per_processor": 15.15, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.7059, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "17c404ea74de2b10", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11227419, "90.00 percentile latency (ns)": 13519919, "90th percentile latency (ns)": 13519919, "95.00 percentile latency (ns)": 21320382, "97.00 percentile latency (ns)": 32095176, "99.00 percentile latency (ns)": 84510013, "99.90 percentile latency (ns)": 107174909, "Max latency (ns)": 121204556, "Mean latency (ns)": 13308576, "Min duration satisfied": "Yes", "Min latency (ns)": 10308968, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 74.95, "QPS w/o loadgen overhead": 75.14, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.08, "characteristics.good": 32040, "characteristics.samples_per_second": 75.14, "characteristics.samples_per_second.normalized_per_core": 12.523333333333333, "characteristics.samples_per_second.normalized_per_processor": 75.14, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "3660fb88c794a588", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20220711, "90.00 percentile latency (ns)": 88496231, "90th percentile latency (ns)": 88496231, "95.00 percentile latency (ns)": 159634421, "97.00 percentile latency (ns)": 174246441, "99.00 percentile latency (ns)": 184792827, "99.90 percentile latency (ns)": 197817499, "Max latency (ns)": 217539693, "Mean latency (ns)": 37623406, "Min duration satisfied": "Yes", "Min latency (ns)": 18862706, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 26.55, "QPS w/o loadgen overhead": 26.58, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.282, "characteristics.good": 32141, "characteristics.samples_per_second": 26.58, "characteristics.samples_per_second.normalized_per_core": 4.43, "characteristics.samples_per_second.normalized_per_processor": 26.58, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 25.641, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "4da6804511d04d5e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 40584579, "90.00 percentile latency (ns)": 136763448, "90th percentile latency (ns)": 136763448, "95.00 percentile latency (ns)": 178781212, "97.00 percentile latency (ns)": 185917151, "99.00 percentile latency (ns)": 194406141, "99.90 percentile latency (ns)": 206307694, "Max latency (ns)": 212562703, "Mean latency (ns)": 61550240, "Min duration satisfied": "Yes", "Min latency (ns)": 37729919, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.24, "QPS w/o loadgen overhead": 16.25, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.678, "characteristics.good": 36339, "characteristics.samples_per_second": 16.25, "characteristics.samples_per_second.normalized_per_core": 2.7083333333333335, "characteristics.samples_per_second.normalized_per_processor": 16.25, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 13.5135, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "5c8f16c826849738", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 13515873, "90.00 percentile latency (ns)": 15632213, "90th percentile latency (ns)": 15632213, "95.00 percentile latency (ns)": 26927257, "97.00 percentile latency (ns)": 38393193, "99.00 percentile latency (ns)": 78908592, "99.90 percentile latency (ns)": 182317462, "Max latency (ns)": 200299919, "Mean latency (ns)": 15952584, "Min duration satisfied": "Yes", "Min latency (ns)": 12071554, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 62.56, "QPS w/o loadgen overhead": 62.69, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.972, "characteristics.good": 32486, "characteristics.samples_per_second": 62.69, "characteristics.samples_per_second.normalized_per_core": 10.448333333333332, "characteristics.samples_per_second.normalized_per_processor": 62.69, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 41.6667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "8c48c0a0e0840c22", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 55579410, "90.00 percentile latency (ns)": 130705422, "90th percentile latency (ns)": 130705422, "95.00 percentile latency (ns)": 194662733, "97.00 percentile latency (ns)": 209757384, "99.00 percentile latency (ns)": 221322568, "99.90 percentile latency (ns)": 231422724, "Max latency (ns)": 241701021, "Mean latency (ns)": 72978069, "Min duration satisfied": "Yes", "Min latency (ns)": 50042207, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 13.7, "QPS w/o loadgen overhead": 13.7, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.77, "characteristics.good": 35885, "characteristics.samples_per_second": 13.7, "characteristics.samples_per_second.normalized_per_core": 2.283333333333333, "characteristics.samples_per_second.normalized_per_processor": 13.7, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 11.4943, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "9516f9421eb55f1d", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 28104014, "90.00 percentile latency (ns)": 167782405, "90th percentile latency (ns)": 167782405, "95.00 percentile latency (ns)": 184891050, "97.00 percentile latency (ns)": 189879478, "99.00 percentile latency (ns)": 196898045, "99.90 percentile latency (ns)": 207462820, "Max latency (ns)": 224065580, "Mean latency (ns)": 57069109, "Min duration satisfied": "Yes", "Min latency (ns)": 22335162, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.51, "QPS w/o loadgen overhead": 17.52, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.75, "characteristics.good": 33375, "characteristics.samples_per_second": 17.52, "characteristics.samples_per_second.normalized_per_core": 2.92, "characteristics.samples_per_second.normalized_per_processor": 17.52, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 21.2766, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "c49ad579e492d024", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10114420, "90.00 percentile latency (ns)": 74626215, "90th percentile latency (ns)": 74626215, "95.00 percentile latency (ns)": 100134153, "97.00 percentile latency (ns)": 119094772, "99.00 percentile latency (ns)": 140332201, "99.90 percentile latency (ns)": 159598885, "Max latency (ns)": 174876717, "Mean latency (ns)": 25676260, "Min duration satisfied": "Yes", "Min latency (ns)": 6165537, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 38.86, "QPS w/o loadgen overhead": 38.95, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 47.14, "characteristics.good": 23570, "characteristics.samples_per_second": 38.95, "characteristics.samples_per_second.normalized_per_core": 6.491666666666667, "characteristics.samples_per_second.normalized_per_processor": 38.95, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "6a161c775bc03dd5", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 18858132, "90.00 percentile latency (ns)": 146217739, "90th percentile latency (ns)": 146217739, "95.00 percentile latency (ns)": 163071607, "97.00 percentile latency (ns)": 169737105, "99.00 percentile latency (ns)": 178508725, "99.90 percentile latency (ns)": 191660267, "Max latency (ns)": 239562484, "Mean latency (ns)": 46976665, "Min duration satisfied": "Yes", "Min latency (ns)": 11400664, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 21.26, "QPS w/o loadgen overhead": 21.29, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.988, "characteristics.good": 28494, "characteristics.samples_per_second": 21.29, "characteristics.samples_per_second.normalized_per_core": 3.5483333333333333, "characteristics.samples_per_second.normalized_per_processor": 21.29, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.5185, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "9d343ca64570a548", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 31269880, "90.00 percentile latency (ns)": 105890323, "90th percentile latency (ns)": 105890323, "95.00 percentile latency (ns)": 109653984, "97.00 percentile latency (ns)": 111758649, "99.00 percentile latency (ns)": 115922671, "99.90 percentile latency (ns)": 124626965, "Max latency (ns)": 141015695, "Mean latency (ns)": 50104368, "Min duration satisfied": "Yes", "Min latency (ns)": 15566977, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 19.93, "QPS w/o loadgen overhead": 19.96, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.716, "characteristics.good": 31858, "characteristics.samples_per_second": 19.96, "characteristics.samples_per_second.normalized_per_core": 3.3266666666666667, "characteristics.samples_per_second.normalized_per_processor": 19.96, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 17.2414, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "ebe3712537e3cfe6", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6553749, "90.00 percentile latency (ns)": 7134169, "90th percentile latency (ns)": 7134169, "95.00 percentile latency (ns)": 8682334, "97.00 percentile latency (ns)": 10252375, "99.00 percentile latency (ns)": 34875174, "99.90 percentile latency (ns)": 164025206, "Max latency (ns)": 192250966, "Mean latency (ns)": 7629007, "Min duration satisfied": "Yes", "Min latency (ns)": 5639085, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 130.49, "QPS w/o loadgen overhead": 131.08, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 55.54, "characteristics.good": 27770, "characteristics.samples_per_second": 131.08, "characteristics.samples_per_second.normalized_per_core": 21.846666666666668, "characteristics.samples_per_second.normalized_per_processor": 131.08, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "ef087ad42a3a27c3", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 21515470, "90.00 percentile latency (ns)": 148947822, "90th percentile latency (ns)": 148947822, "95.00 percentile latency (ns)": 168945597, "97.00 percentile latency (ns)": 174662026, "99.00 percentile latency (ns)": 181346891, "99.90 percentile latency (ns)": 192841867, "Max latency (ns)": 203763581, "Mean latency (ns)": 49312902, "Min duration satisfied": "Yes", "Min latency (ns)": 14944785, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.26, "QPS w/o loadgen overhead": 20.28, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.322, "characteristics.good": 29661, "characteristics.samples_per_second": 20.28, "characteristics.samples_per_second.normalized_per_core": 3.3800000000000003, "characteristics.samples_per_second.normalized_per_processor": 20.28, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.625, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "361a4727b1d1ba65", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 34575341, "90.00 percentile latency (ns)": 35067145, "90th percentile latency (ns)": 35067145, "95.00 percentile latency (ns)": 36570177, "97.00 percentile latency (ns)": 46531923, "99.00 percentile latency (ns)": 60195429, "99.90 percentile latency (ns)": 109382166, "Max latency (ns)": 124320023, "Mean latency (ns)": 35460325, "Min duration satisfied": "Yes", "Min latency (ns)": 33674769, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 28.18, "QPS w/o loadgen overhead": 28.2, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.242, "characteristics.good": 34121, "characteristics.samples_per_second": 28.2, "characteristics.samples_per_second.normalized_per_core": 7.05, "characteristics.samples_per_second.normalized_per_processor": 28.2, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 27.027, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "a6a97914f6986c71", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 41990312, "90.00 percentile latency (ns)": 42502252, "90th percentile latency (ns)": 42502252, "95.00 percentile latency (ns)": 44559768, "97.00 percentile latency (ns)": 60733015, "99.00 percentile latency (ns)": 71329224, "99.90 percentile latency (ns)": 156436018, "Max latency (ns)": 169781847, "Mean latency (ns)": 43189394, "Min duration satisfied": "Yes", "Min latency (ns)": 41064835, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 23.14, "QPS w/o loadgen overhead": 23.15, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.738, "characteristics.good": 35369, "characteristics.samples_per_second": 23.15, "characteristics.samples_per_second.normalized_per_core": 5.7875, "characteristics.samples_per_second.normalized_per_processor": 23.15, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 22.2222, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "5b21fc06c904ba9a", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 162092451, "90.00 percentile latency (ns)": 182694593, "90th percentile latency (ns)": 182694593, "95.00 percentile latency (ns)": 185520236, "97.00 percentile latency (ns)": 187028054, "99.00 percentile latency (ns)": 196597234, "99.90 percentile latency (ns)": 299178325, "Max latency (ns)": 307633662, "Mean latency (ns)": 165746355, "Min duration satisfied": "Yes", "Min latency (ns)": 160430275, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 6.03, "QPS w/o loadgen overhead": 6.03, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 78.514, "characteristics.good": 39257, "characteristics.samples_per_second": 6.03, "characteristics.samples_per_second.normalized_per_core": 1.5075, "characteristics.samples_per_second.normalized_per_processor": 6.03, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite4-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 5.91716, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "4f35a230b75c4809", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 12042603, "90.00 percentile latency (ns)": 12822551, "90th percentile latency (ns)": 12822551, "95.00 percentile latency (ns)": 13038380, "97.00 percentile latency (ns)": 13163692, "99.00 percentile latency (ns)": 29963640, "99.90 percentile latency (ns)": 47836677, "Max latency (ns)": 129364113, "Mean latency (ns)": 12546515, "Min duration satisfied": "Yes", "Min latency (ns)": 11146085, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 79.5, "QPS w/o loadgen overhead": 79.7, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.542, "characteristics.good": 31771, "characteristics.samples_per_second": 79.7, "characteristics.samples_per_second.normalized_per_core": 19.925, "characteristics.samples_per_second.normalized_per_processor": 79.7, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "334088b59518edeb", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8427168, "90.00 percentile latency (ns)": 8959461, "90th percentile latency (ns)": 8959461, "95.00 percentile latency (ns)": 9105531, "97.00 percentile latency (ns)": 9241173, "99.00 percentile latency (ns)": 12785740, "99.90 percentile latency (ns)": 44216719, "Max latency (ns)": 121545854, "Mean latency (ns)": 8737590, "Min duration satisfied": "Yes", "Min latency (ns)": 7565002, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 113.88, "QPS w/o loadgen overhead": 114.45, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 52.172, "characteristics.good": 26086, "characteristics.samples_per_second": 114.45, "characteristics.samples_per_second.normalized_per_core": 28.6125, "characteristics.samples_per_second.normalized_per_processor": 114.45, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "832aa48fed30ffe6", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10233682, "90.00 percentile latency (ns)": 10890783, "90th percentile latency (ns)": 10890783, "95.00 percentile latency (ns)": 11004552, "97.00 percentile latency (ns)": 11090738, "99.00 percentile latency (ns)": 18851660, "99.90 percentile latency (ns)": 49247363, "Max latency (ns)": 127791980, "Mean latency (ns)": 10655567, "Min duration satisfied": "Yes", "Min latency (ns)": 9430542, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 93.56, "QPS w/o loadgen overhead": 93.85, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.402, "characteristics.good": 29201, "characteristics.samples_per_second": 93.85, "characteristics.samples_per_second.normalized_per_core": 23.4625, "characteristics.samples_per_second.normalized_per_processor": 93.85, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "24d668c6dec74679", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 21720922, "90.00 percentile latency (ns)": 22050171, "90th percentile latency (ns)": 22050171, "95.00 percentile latency (ns)": 22268965, "97.00 percentile latency (ns)": 23652768, "99.00 percentile latency (ns)": 50511591, "99.90 percentile latency (ns)": 95245518, "Max latency (ns)": 149496854, "Mean latency (ns)": 22385494, "Min duration satisfied": "Yes", "Min latency (ns)": 21079696, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 44.63, "QPS w/o loadgen overhead": 44.67, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 60.976, "characteristics.good": 30488, "characteristics.samples_per_second": 44.67, "characteristics.samples_per_second.normalized_per_core": 11.1675, "characteristics.samples_per_second.normalized_per_processor": 44.67, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 43.4783, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "b4c939271bc26907", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3974564, "90.00 percentile latency (ns)": 4691901, "90th percentile latency (ns)": 4691901, "95.00 percentile latency (ns)": 4863156, "97.00 percentile latency (ns)": 4949507, "99.00 percentile latency (ns)": 5169205, "99.90 percentile latency (ns)": 36868462, "Max latency (ns)": 77357768, "Mean latency (ns)": 4222208, "Min duration satisfied": "Yes", "Min latency (ns)": 3406576, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 234.76, "QPS w/o loadgen overhead": 236.84, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 48.162, "characteristics.good": 24081, "characteristics.samples_per_second": 236.84, "characteristics.samples_per_second.normalized_per_core": 59.21, "characteristics.samples_per_second.normalized_per_processor": 236.84, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "46ac996e9a401372", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 67664091, "90.00 percentile latency (ns)": 68728100, "90th percentile latency (ns)": 68728100, "95.00 percentile latency (ns)": 82694627, "97.00 percentile latency (ns)": 92542461, "99.00 percentile latency (ns)": 96499362, "99.90 percentile latency (ns)": 184719690, "Max latency (ns)": 192401399, "Mean latency (ns)": 69408019, "Min duration satisfied": "Yes", "Min latency (ns)": 66203908, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 14.4, "QPS w/o loadgen overhead": 14.41, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.068, "characteristics.good": 37534, "characteristics.samples_per_second": 14.41, "characteristics.samples_per_second.normalized_per_core": 3.6025, "characteristics.samples_per_second.normalized_per_processor": 14.41, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite0-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.2857, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "cd467dfc5c54050b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5170703, "90.00 percentile latency (ns)": 5760347, "90th percentile latency (ns)": 5760347, "95.00 percentile latency (ns)": 5845417, "97.00 percentile latency (ns)": 5987713, "99.00 percentile latency (ns)": 6366979, "99.90 percentile latency (ns)": 38316674, "Max latency (ns)": 81320842, "Mean latency (ns)": 5420017, "Min duration satisfied": "Yes", "Min latency (ns)": 4491685, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 182.93, "QPS w/o loadgen overhead": 184.5, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 50.242, "characteristics.good": 25121, "characteristics.samples_per_second": 184.5, "characteristics.samples_per_second.normalized_per_core": 46.125, "characteristics.samples_per_second.normalized_per_processor": 184.5, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "4802abd445c7cdc4", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 14559803, "90.00 percentile latency (ns)": 14886336, "90th percentile latency (ns)": 14886336, "95.00 percentile latency (ns)": 15050347, "97.00 percentile latency (ns)": 15399911, "99.00 percentile latency (ns)": 36829958, "99.90 percentile latency (ns)": 72604231, "Max latency (ns)": 98686002, "Mean latency (ns)": 14981959, "Min duration satisfied": "Yes", "Min latency (ns)": 13866986, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 66.65, "QPS w/o loadgen overhead": 66.75, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 60.842, "characteristics.good": 30421, "characteristics.samples_per_second": 66.75, "characteristics.samples_per_second.normalized_per_core": 16.6875, "characteristics.samples_per_second.normalized_per_processor": 66.75, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "7ae5bf620056f64e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 31831370, "90.00 percentile latency (ns)": 32344671, "90th percentile latency (ns)": 32344671, "95.00 percentile latency (ns)": 32948824, "97.00 percentile latency (ns)": 42498263, "99.00 percentile latency (ns)": 61671090, "99.90 percentile latency (ns)": 132842784, "Max latency (ns)": 166811257, "Mean latency (ns)": 32789503, "Min duration satisfied": "Yes", "Min latency (ns)": 31003200, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 30.48, "QPS w/o loadgen overhead": 30.5, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.016, "characteristics.good": 34508, "characteristics.samples_per_second": 30.5, "characteristics.samples_per_second.normalized_per_core": 7.625, "characteristics.samples_per_second.normalized_per_processor": 30.5, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 28.5714, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "4e15a791c666a4cc", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 46756209, "90.00 percentile latency (ns)": 47387771, "90th percentile latency (ns)": 47387771, "95.00 percentile latency (ns)": 50937863, "97.00 percentile latency (ns)": 72091760, "99.00 percentile latency (ns)": 76413065, "99.90 percentile latency (ns)": 164039828, "Max latency (ns)": 196242063, "Mean latency (ns)": 48023536, "Min duration satisfied": "Yes", "Min latency (ns)": 45416955, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.81, "QPS w/o loadgen overhead": 20.82, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.098, "characteristics.good": 37049, "characteristics.samples_per_second": 20.82, "characteristics.samples_per_second.normalized_per_core": 5.205, "characteristics.samples_per_second.normalized_per_processor": 20.82, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite1-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 20.4082, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "20b28f4b89a7f973", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9300569, "90.00 percentile latency (ns)": 9839735, "90th percentile latency (ns)": 9839735, "95.00 percentile latency (ns)": 9986101, "97.00 percentile latency (ns)": 10103655, "99.00 percentile latency (ns)": 15433810, "99.90 percentile latency (ns)": 45212863, "Max latency (ns)": 129642837, "Mean latency (ns)": 9677034, "Min duration satisfied": "Yes", "Min latency (ns)": 8551982, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 103.06, "QPS w/o loadgen overhead": 103.34, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.924, "characteristics.good": 30962, "characteristics.samples_per_second": 103.34, "characteristics.samples_per_second.normalized_per_core": 25.835, "characteristics.samples_per_second.normalized_per_processor": 103.34, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "3898555a76241038", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10450859, "90.00 percentile latency (ns)": 11085027, "90th percentile latency (ns)": 11085027, "95.00 percentile latency (ns)": 11202529, "97.00 percentile latency (ns)": 11436019, "99.00 percentile latency (ns)": 23366240, "99.90 percentile latency (ns)": 50006687, "Max latency (ns)": 92572933, "Mean latency (ns)": 10865066, "Min duration satisfied": "Yes", "Min latency (ns)": 9721962, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 91.76, "QPS w/o loadgen overhead": 92.04, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 62.298, "characteristics.good": 31149, "characteristics.samples_per_second": 92.04, "characteristics.samples_per_second.normalized_per_core": 23.01, "characteristics.samples_per_second.normalized_per_processor": 92.04, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "a2a5a12e4e2e8b00", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4164915, "90.00 percentile latency (ns)": 4799920, "90th percentile latency (ns)": 4799920, "95.00 percentile latency (ns)": 4964637, "97.00 percentile latency (ns)": 5107283, "99.00 percentile latency (ns)": 5385924, "99.90 percentile latency (ns)": 37704345, "Max latency (ns)": 118063193, "Mean latency (ns)": 4423729, "Min duration satisfied": "Yes", "Min latency (ns)": 3733480, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 223.93, "QPS w/o loadgen overhead": 226.05, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.12, "characteristics.good": 24560, "characteristics.samples_per_second": 226.05, "characteristics.samples_per_second.normalized_per_core": 56.5125, "characteristics.samples_per_second.normalized_per_processor": 226.05, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "4777bf69847ab741", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 39569493, "90.00 percentile latency (ns)": 40158040, "90th percentile latency (ns)": 40158040, "95.00 percentile latency (ns)": 40961886, "97.00 percentile latency (ns)": 57958025, "99.00 percentile latency (ns)": 67856257, "99.90 percentile latency (ns)": 133270824, "Max latency (ns)": 169298856, "Mean latency (ns)": 40634399, "Min duration satisfied": "Yes", "Min latency (ns)": 38249700, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 24.59, "QPS w/o loadgen overhead": 24.61, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.584, "characteristics.good": 36792, "characteristics.samples_per_second": 24.61, "characteristics.samples_per_second.normalized_per_core": 6.1525, "characteristics.samples_per_second.normalized_per_processor": 24.61, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.4-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.4-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_140.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 24.3902, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "ba38fe30fbee7296", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 28523266, "90.00 percentile latency (ns)": 29170159, "90th percentile latency (ns)": 29170159, "95.00 percentile latency (ns)": 29445384, "97.00 percentile latency (ns)": 36745900, "99.00 percentile latency (ns)": 54293374, "99.90 percentile latency (ns)": 100987466, "Max latency (ns)": 129260598, "Mean latency (ns)": 29323948, "Min duration satisfied": "Yes", "Min latency (ns)": 27540484, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 34.07, "QPS w/o loadgen overhead": 34.1, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.772, "characteristics.good": 35386, "characteristics.samples_per_second": 34.1, "characteristics.samples_per_second.normalized_per_core": 8.525, "characteristics.samples_per_second.normalized_per_processor": 34.1, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 32.2581, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "285897a5428c9b1f", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7198060, "90.00 percentile latency (ns)": 7441224, "90th percentile latency (ns)": 7441224, "95.00 percentile latency (ns)": 7528257, "97.00 percentile latency (ns)": 7596182, "99.00 percentile latency (ns)": 8960806, "99.90 percentile latency (ns)": 41811556, "Max latency (ns)": 81812146, "Mean latency (ns)": 7422100, "Min duration satisfied": "Yes", "Min latency (ns)": 6579164, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 134.39, "QPS w/o loadgen overhead": 134.73, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 47.808, "characteristics.good": 23904, "characteristics.samples_per_second": 134.73, "characteristics.samples_per_second.normalized_per_core": 33.6825, "characteristics.samples_per_second.normalized_per_processor": 134.73, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "3d985d183e45bf94", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 16766070, "90.00 percentile latency (ns)": 17421359, "90th percentile latency (ns)": 17421359, "95.00 percentile latency (ns)": 17622700, "97.00 percentile latency (ns)": 18065109, "99.00 percentile latency (ns)": 42251845, "99.90 percentile latency (ns)": 76600673, "Max latency (ns)": 150810689, "Mean latency (ns)": 17346805, "Min duration satisfied": "Yes", "Min latency (ns)": 15801037, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 57.53, "QPS w/o loadgen overhead": 57.65, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.038, "characteristics.good": 32019, "characteristics.samples_per_second": 57.65, "characteristics.samples_per_second.normalized_per_core": 14.4125, "characteristics.samples_per_second.normalized_per_processor": 57.65, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 55.5556, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "88a26af9a0ef3207", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 13421761, "90.00 percentile latency (ns)": 13698625, "90th percentile latency (ns)": 13698625, "95.00 percentile latency (ns)": 13804327, "97.00 percentile latency (ns)": 14021989, "99.00 percentile latency (ns)": 35096555, "99.90 percentile latency (ns)": 67775585, "Max latency (ns)": 97925574, "Mean latency (ns)": 13829230, "Min duration satisfied": "Yes", "Min latency (ns)": 12819050, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 72.2, "QPS w/o loadgen overhead": 72.31, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 51.848, "characteristics.good": 25924, "characteristics.samples_per_second": 72.31, "characteristics.samples_per_second.normalized_per_core": 18.0775, "characteristics.samples_per_second.normalized_per_processor": 72.31, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "8906f0f5e20f9f14", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7328832, "90.00 percentile latency (ns)": 7960636, "90th percentile latency (ns)": 7960636, "95.00 percentile latency (ns)": 8134555, "97.00 percentile latency (ns)": 8260674, "99.00 percentile latency (ns)": 9970528, "99.90 percentile latency (ns)": 40666120, "Max latency (ns)": 118481831, "Mean latency (ns)": 7646349, "Min duration satisfied": "Yes", "Min latency (ns)": 6610371, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 130.24, "QPS w/o loadgen overhead": 130.78, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.226, "characteristics.good": 29613, "characteristics.samples_per_second": 130.78, "characteristics.samples_per_second.normalized_per_core": 32.695, "characteristics.samples_per_second.normalized_per_processor": 130.78, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "df54c86b324f2042", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 15872213, "90.00 percentile latency (ns)": 16442220, "90th percentile latency (ns)": 16442220, "95.00 percentile latency (ns)": 16640545, "97.00 percentile latency (ns)": 16966543, "99.00 percentile latency (ns)": 38451482, "99.90 percentile latency (ns)": 74839651, "Max latency (ns)": 90416649, "Mean latency (ns)": 16395777, "Min duration satisfied": "Yes", "Min latency (ns)": 15057858, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 60.89, "QPS w/o loadgen overhead": 60.99, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.248, "characteristics.good": 34124, "characteristics.samples_per_second": 60.99, "characteristics.samples_per_second.normalized_per_core": 15.2475, "characteristics.samples_per_second.normalized_per_processor": 60.99, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 55.5556, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "f4eb9824578fe309", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6415153, "90.00 percentile latency (ns)": 7121232, "90th percentile latency (ns)": 7121232, "95.00 percentile latency (ns)": 7270980, "97.00 percentile latency (ns)": 7406394, "99.00 percentile latency (ns)": 8449677, "99.90 percentile latency (ns)": 40466557, "Max latency (ns)": 122506380, "Mean latency (ns)": 6763174, "Min duration satisfied": "Yes", "Min latency (ns)": 5856571, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 147.04, "QPS w/o loadgen overhead": 147.86, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.12, "characteristics.good": 29560, "characteristics.samples_per_second": 147.86, "characteristics.samples_per_second.normalized_per_core": 36.965, "characteristics.samples_per_second.normalized_per_processor": 147.86, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "211447b447ca90cb", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7014627, "90.00 percentile latency (ns)": 7665856, "90th percentile latency (ns)": 7665856, "95.00 percentile latency (ns)": 7795516, "97.00 percentile latency (ns)": 7934297, "99.00 percentile latency (ns)": 11482872, "99.90 percentile latency (ns)": 40186819, "Max latency (ns)": 85614421, "Mean latency (ns)": 7342133, "Min duration satisfied": "Yes", "Min latency (ns)": 6463637, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 135.54, "QPS w/o loadgen overhead": 136.2, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.46, "characteristics.good": 29230, "characteristics.samples_per_second": 136.2, "characteristics.samples_per_second.normalized_per_core": 34.05, "characteristics.samples_per_second.normalized_per_processor": 136.2, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "5eb4dc4242fc950e", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 26387339, "90.00 percentile latency (ns)": 27060358, "90th percentile latency (ns)": 27060358, "95.00 percentile latency (ns)": 27372410, "97.00 percentile latency (ns)": 31150265, "99.00 percentile latency (ns)": 54631213, "99.90 percentile latency (ns)": 107314416, "Max latency (ns)": 148023456, "Mean latency (ns)": 27203248, "Min duration satisfied": "Yes", "Min latency (ns)": 25360967, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 36.71, "QPS w/o loadgen overhead": 36.76, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.91, "characteristics.good": 36455, "characteristics.samples_per_second": 36.76, "characteristics.samples_per_second.normalized_per_core": 9.19, "characteristics.samples_per_second.normalized_per_processor": 36.76, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 35.7143, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "536b7e76d4efea54", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 24031350, "90.00 percentile latency (ns)": 24679691, "90th percentile latency (ns)": 24679691, "95.00 percentile latency (ns)": 25015696, "97.00 percentile latency (ns)": 27186625, "99.00 percentile latency (ns)": 52576458, "99.90 percentile latency (ns)": 102878653, "Max latency (ns)": 156155783, "Mean latency (ns)": 24821674, "Min duration satisfied": "Yes", "Min latency (ns)": 23307981, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 40.23, "QPS w/o loadgen overhead": 40.29, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.542, "characteristics.good": 35271, "characteristics.samples_per_second": 40.29, "characteristics.samples_per_second.normalized_per_core": 10.0725, "characteristics.samples_per_second.normalized_per_processor": 40.29, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 38.4615, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "36adbe61470da8ec", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7443421, "90.00 percentile latency (ns)": 7991429, "90th percentile latency (ns)": 7991429, "95.00 percentile latency (ns)": 8106403, "97.00 percentile latency (ns)": 8232666, "99.00 percentile latency (ns)": 10562265, "99.90 percentile latency (ns)": 41953602, "Max latency (ns)": 123320932, "Mean latency (ns)": 7757026, "Min duration satisfied": "Yes", "Min latency (ns)": 6706114, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 128.36, "QPS w/o loadgen overhead": 128.92, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.594, "characteristics.good": 28297, "characteristics.samples_per_second": 128.92, "characteristics.samples_per_second.normalized_per_core": 32.23, "characteristics.samples_per_second.normalized_per_processor": 128.92, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "9f474e92f17367d2", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 13599398, "90.00 percentile latency (ns)": 14252029, "90th percentile latency (ns)": 14252029, "95.00 percentile latency (ns)": 14472820, "97.00 percentile latency (ns)": 14626802, "99.00 percentile latency (ns)": 36730180, "99.90 percentile latency (ns)": 52897911, "Max latency (ns)": 137741048, "Mean latency (ns)": 14112140, "Min duration satisfied": "Yes", "Min latency (ns)": 12747671, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 70.72, "QPS w/o loadgen overhead": 70.86, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.82, "characteristics.good": 33910, "characteristics.samples_per_second": 70.86, "characteristics.samples_per_second.normalized_per_core": 17.715, "characteristics.samples_per_second.normalized_per_processor": 70.86, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "80a708d442128eb2", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4710108, "90.00 percentile latency (ns)": 5222671, "90th percentile latency (ns)": 5222671, "95.00 percentile latency (ns)": 5438481, "97.00 percentile latency (ns)": 5551312, "99.00 percentile latency (ns)": 6221706, "99.90 percentile latency (ns)": 39752852, "Max latency (ns)": 82402588, "Mean latency (ns)": 4922644, "Min duration satisfied": "Yes", "Min latency (ns)": 4096213, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 201.52, "QPS w/o loadgen overhead": 203.14, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 44.116, "characteristics.good": 22058, "characteristics.samples_per_second": 203.14, "characteristics.samples_per_second.normalized_per_core": 50.785, "characteristics.samples_per_second.normalized_per_processor": 203.14, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "43ab9c43022dc81e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2984470, "90.00 percentile latency (ns)": 3590441, "90th percentile latency (ns)": 3590441, "95.00 percentile latency (ns)": 3745604, "97.00 percentile latency (ns)": 3935710, "99.00 percentile latency (ns)": 4146539, "99.90 percentile latency (ns)": 35560513, "Max latency (ns)": 72447002, "Mean latency (ns)": 3173152, "Min duration satisfied": "Yes", "Min latency (ns)": 2506276, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 311.24, "QPS w/o loadgen overhead": 315.14, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 45.494, "characteristics.good": 22747, "characteristics.samples_per_second": 315.14, "characteristics.samples_per_second.normalized_per_core": 78.785, "characteristics.samples_per_second.normalized_per_processor": 315.14, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "00256044ebebab90", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 52569062, "90.00 percentile latency (ns)": 53315044, "90th percentile latency (ns)": 53315044, "95.00 percentile latency (ns)": 61573380, "97.00 percentile latency (ns)": 69989332, "99.00 percentile latency (ns)": 78454509, "99.90 percentile latency (ns)": 132094300, "Max latency (ns)": 153030446, "Mean latency (ns)": 53791603, "Min duration satisfied": "Yes", "Min latency (ns)": 51267942, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 18.58, "QPS w/o loadgen overhead": 18.59, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.626, "characteristics.good": 35313, "characteristics.samples_per_second": 18.59, "characteristics.samples_per_second.normalized_per_core": 4.6475, "characteristics.samples_per_second.normalized_per_processor": 18.59, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 17.8571, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "f3a4793b0aecea08", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9338868, "90.00 percentile latency (ns)": 9898754, "90th percentile latency (ns)": 9898754, "95.00 percentile latency (ns)": 10043472, "97.00 percentile latency (ns)": 10165187, "99.00 percentile latency (ns)": 16680091, "99.90 percentile latency (ns)": 45878728, "Max latency (ns)": 142598327, "Mean latency (ns)": 9724423, "Min duration satisfied": "Yes", "Min latency (ns)": 8629957, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 102.52, "QPS w/o loadgen overhead": 102.83, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.454, "characteristics.good": 31727, "characteristics.samples_per_second": 102.83, "characteristics.samples_per_second.normalized_per_core": 25.7075, "characteristics.samples_per_second.normalized_per_processor": 102.83, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "65844b1a1698023a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 21434494, "90.00 percentile latency (ns)": 22059354, "90th percentile latency (ns)": 22059354, "95.00 percentile latency (ns)": 22327123, "97.00 percentile latency (ns)": 23871663, "99.00 percentile latency (ns)": 47152124, "99.90 percentile latency (ns)": 82661789, "Max latency (ns)": 112760963, "Mean latency (ns)": 22069422, "Min duration satisfied": "Yes", "Min latency (ns)": 20505149, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 45.2, "QPS w/o loadgen overhead": 45.31, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.04, "characteristics.good": 35020, "characteristics.samples_per_second": 45.31, "characteristics.samples_per_second.normalized_per_core": 11.3275, "characteristics.samples_per_second.normalized_per_processor": 45.31, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 43.4783, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "a18d89746558918f", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 63829267, "90.00 percentile latency (ns)": 64608132, "90th percentile latency (ns)": 64608132, "95.00 percentile latency (ns)": 81285867, "97.00 percentile latency (ns)": 88853366, "99.00 percentile latency (ns)": 91467600, "99.90 percentile latency (ns)": 184169896, "Max latency (ns)": 214979046, "Mean latency (ns)": 65449195, "Min duration satisfied": "Yes", "Min latency (ns)": 62406231, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 15.27, "QPS w/o loadgen overhead": 15.28, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.124, "characteristics.good": 37062, "characteristics.samples_per_second": 15.28, "characteristics.samples_per_second.normalized_per_core": 3.82, "characteristics.samples_per_second.normalized_per_processor": 15.28, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite2-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.9254, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "2585f24b898f3d05", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5990726, "90.00 percentile latency (ns)": 6638291, "90th percentile latency (ns)": 6638291, "95.00 percentile latency (ns)": 6757048, "97.00 percentile latency (ns)": 6882059, "99.00 percentile latency (ns)": 7858596, "99.90 percentile latency (ns)": 39165756, "Max latency (ns)": 79187662, "Mean latency (ns)": 6278313, "Min duration satisfied": "Yes", "Min latency (ns)": 5434438, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 158.36, "QPS w/o loadgen overhead": 159.28, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.444, "characteristics.good": 29722, "characteristics.samples_per_second": 159.28, "characteristics.samples_per_second.normalized_per_core": 39.82, "characteristics.samples_per_second.normalized_per_processor": 159.28, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "c1442c3a3a911f1b", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8056815, "90.00 percentile latency (ns)": 8755910, "90th percentile latency (ns)": 8755910, "95.00 percentile latency (ns)": 8900888, "97.00 percentile latency (ns)": 8998942, "99.00 percentile latency (ns)": 13753866, "99.90 percentile latency (ns)": 41350696, "Max latency (ns)": 86070410, "Mean latency (ns)": 8419986, "Min duration satisfied": "Yes", "Min latency (ns)": 7341608, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 118.38, "QPS w/o loadgen overhead": 118.77, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.456, "characteristics.good": 30728, "characteristics.samples_per_second": 118.77, "characteristics.samples_per_second.normalized_per_core": 29.6925, "characteristics.samples_per_second.normalized_per_processor": 118.77, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "ece884fc9512c2da", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9595900, "90.00 percentile latency (ns)": 10214761, "90th percentile latency (ns)": 10214761, "95.00 percentile latency (ns)": 10401645, "97.00 percentile latency (ns)": 10509902, "99.00 percentile latency (ns)": 15822954, "99.90 percentile latency (ns)": 45006282, "Max latency (ns)": 151492909, "Mean latency (ns)": 10000702, "Min duration satisfied": "Yes", "Min latency (ns)": 8760051, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 99.72, "QPS w/o loadgen overhead": 99.99, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.528, "characteristics.good": 29264, "characteristics.samples_per_second": 99.99, "characteristics.samples_per_second.normalized_per_core": 24.9975, "characteristics.samples_per_second.normalized_per_processor": 99.99, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "54245ddbbc9bc68c", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 21965984, "90.00 percentile latency (ns)": 22641562, "90th percentile latency (ns)": 22641562, "95.00 percentile latency (ns)": 22974334, "97.00 percentile latency (ns)": 26597594, "99.00 percentile latency (ns)": 52437351, "99.90 percentile latency (ns)": 136752654, "Max latency (ns)": 151250797, "Mean latency (ns)": 22923624, "Min duration satisfied": "Yes", "Min latency (ns)": 21027633, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 43.55, "QPS w/o loadgen overhead": 43.62, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.036, "characteristics.good": 33018, "characteristics.samples_per_second": 43.62, "characteristics.samples_per_second.normalized_per_core": 10.905, "characteristics.samples_per_second.normalized_per_processor": 43.62, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 43.4783, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "a39cbc6868c61518", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 95420062, "90.00 percentile latency (ns)": 98668026, "90th percentile latency (ns)": 98668026, "95.00 percentile latency (ns)": 117257676, "97.00 percentile latency (ns)": 120272644, "99.00 percentile latency (ns)": 124676952, "99.90 percentile latency (ns)": 249585139, "Max latency (ns)": 260516940, "Mean latency (ns)": 97737335, "Min duration satisfied": "Yes", "Min latency (ns)": 93680137, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 10.23, "QPS w/o loadgen overhead": 10.23, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.642, "characteristics.good": 38321, "characteristics.samples_per_second": 10.23, "characteristics.samples_per_second.normalized_per_core": 2.5575, "characteristics.samples_per_second.normalized_per_processor": 10.23, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite1-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 10.101, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "7540b9b2fdb181bf", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 18379194, "90.00 percentile latency (ns)": 19049235, "90th percentile latency (ns)": 19049235, "95.00 percentile latency (ns)": 19269281, "97.00 percentile latency (ns)": 19602645, "99.00 percentile latency (ns)": 46625847, "99.90 percentile latency (ns)": 82746696, "Max latency (ns)": 155697083, "Mean latency (ns)": 19019574, "Min duration satisfied": "Yes", "Min latency (ns)": 17569183, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 52.44, "QPS w/o loadgen overhead": 52.58, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.604, "characteristics.good": 34802, "characteristics.samples_per_second": 52.58, "characteristics.samples_per_second.normalized_per_core": 13.145, "characteristics.samples_per_second.normalized_per_processor": 52.58, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 52.6316, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "a94e4453e9d0aba8", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 45935552, "90.00 percentile latency (ns)": 46474513, "90th percentile latency (ns)": 46474513, "95.00 percentile latency (ns)": 51214695, "97.00 percentile latency (ns)": 63174585, "99.00 percentile latency (ns)": 71406537, "99.90 percentile latency (ns)": 122483644, "Max latency (ns)": 135690018, "Mean latency (ns)": 47042807, "Min duration satisfied": "Yes", "Min latency (ns)": 44988500, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 21.25, "QPS w/o loadgen overhead": 21.26, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.262, "characteristics.good": 34631, "characteristics.samples_per_second": 21.26, "characteristics.samples_per_second.normalized_per_core": 5.315, "characteristics.samples_per_second.normalized_per_processor": 21.26, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 20, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "b6a1c032431ebe17", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 19981489, "90.00 percentile latency (ns)": 20466345, "90th percentile latency (ns)": 20466345, "95.00 percentile latency (ns)": 20751726, "97.00 percentile latency (ns)": 21154509, "99.00 percentile latency (ns)": 48718113, "99.90 percentile latency (ns)": 79722247, "Max latency (ns)": 149265924, "Mean latency (ns)": 20628925, "Min duration satisfied": "Yes", "Min latency (ns)": 19110480, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 48.35, "QPS w/o loadgen overhead": 48.48, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.734, "characteristics.good": 34367, "characteristics.samples_per_second": 48.48, "characteristics.samples_per_second.normalized_per_core": 12.12, "characteristics.samples_per_second.normalized_per_processor": 48.48, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 47.619, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "9092a16877a7eb1f", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 16432399, "90.00 percentile latency (ns)": 16714764, "90th percentile latency (ns)": 16714764, "95.00 percentile latency (ns)": 16914779, "97.00 percentile latency (ns)": 17295641, "99.00 percentile latency (ns)": 39525974, "99.90 percentile latency (ns)": 89558419, "Max latency (ns)": 112846419, "Mean latency (ns)": 16923876, "Min duration satisfied": "Yes", "Min latency (ns)": 15871949, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 59.01, "QPS w/o loadgen overhead": 59.09, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.33, "characteristics.good": 31665, "characteristics.samples_per_second": 59.09, "characteristics.samples_per_second.normalized_per_core": 14.7725, "characteristics.samples_per_second.normalized_per_processor": 59.09, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 58.8235, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "93e44c6787796366", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9557442, "90.00 percentile latency (ns)": 10106448, "90th percentile latency (ns)": 10106448, "95.00 percentile latency (ns)": 10241705, "97.00 percentile latency (ns)": 10371102, "99.00 percentile latency (ns)": 18982589, "99.90 percentile latency (ns)": 54680545, "Max latency (ns)": 91177085, "Mean latency (ns)": 9910146, "Min duration satisfied": "Yes", "Min latency (ns)": 8684363, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 100.51, "QPS w/o loadgen overhead": 100.91, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 57.918, "characteristics.good": 28959, "characteristics.samples_per_second": 100.91, "characteristics.samples_per_second.normalized_per_core": 25.2275, "characteristics.samples_per_second.normalized_per_processor": 100.91, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "303246557459f9c7", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 68427325, "90.00 percentile latency (ns)": 69552097, "90th percentile latency (ns)": 69552097, "95.00 percentile latency (ns)": 82122609, "97.00 percentile latency (ns)": 89516212, "99.00 percentile latency (ns)": 93838674, "99.90 percentile latency (ns)": 159916364, "Max latency (ns)": 176765064, "Mean latency (ns)": 69911091, "Min duration satisfied": "Yes", "Min latency (ns)": 66086986, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 14.3, "QPS w/o loadgen overhead": 14.3, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.676, "characteristics.good": 35838, "characteristics.samples_per_second": 14.3, "characteristics.samples_per_second.normalized_per_core": 3.575, "characteristics.samples_per_second.normalized_per_processor": 14.3, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.0845, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "8ff66594ecebd670", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3070046, "90.00 percentile latency (ns)": 3559850, "90th percentile latency (ns)": 3559850, "95.00 percentile latency (ns)": 3698866, "97.00 percentile latency (ns)": 3780327, "99.00 percentile latency (ns)": 4052617, "99.90 percentile latency (ns)": 36065807, "Max latency (ns)": 111121191, "Mean latency (ns)": 3242272, "Min duration satisfied": "Yes", "Min latency (ns)": 2484448, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 305.12, "QPS w/o loadgen overhead": 308.43, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 43.468, "characteristics.good": 21734, "characteristics.samples_per_second": 308.43, "characteristics.samples_per_second.normalized_per_core": 77.1075, "characteristics.samples_per_second.normalized_per_processor": 308.43, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "e842de52405291d9", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 330079270, "90.00 percentile latency (ns)": 350427079, "90th percentile latency (ns)": 350427079, "95.00 percentile latency (ns)": 353426040, "97.00 percentile latency (ns)": 357431632, "99.00 percentile latency (ns)": 429357518, "99.90 percentile latency (ns)": 474229017, "Max latency (ns)": 482662101, "Mean latency (ns)": 336223383, "Min duration satisfied": "Yes", "Min latency (ns)": 326448781, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.97, "QPS w/o loadgen overhead": 2.97, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 80.676, "characteristics.good": 40338, "characteristics.samples_per_second": 2.97, "characteristics.samples_per_second.normalized_per_core": 0.7425, "characteristics.samples_per_second.normalized_per_processor": 2.97, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite4-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.92398, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "a5b840ecabcc564f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 18894790, "90.00 percentile latency (ns)": 19465112, "90th percentile latency (ns)": 19465112, "95.00 percentile latency (ns)": 19722347, "97.00 percentile latency (ns)": 20044822, "99.00 percentile latency (ns)": 47347197, "99.90 percentile latency (ns)": 80132253, "Max latency (ns)": 140914622, "Mean latency (ns)": 19533076, "Min duration satisfied": "Yes", "Min latency (ns)": 18179358, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 51.06, "QPS w/o loadgen overhead": 51.2, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.43, "characteristics.good": 35215, "characteristics.samples_per_second": 51.2, "characteristics.samples_per_second.normalized_per_core": 12.8, "characteristics.samples_per_second.normalized_per_processor": 51.2, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "8396fbf5dc541829", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 15256829, "90.00 percentile latency (ns)": 15910566, "90th percentile latency (ns)": 15910566, "95.00 percentile latency (ns)": 16094666, "97.00 percentile latency (ns)": 16249330, "99.00 percentile latency (ns)": 42332926, "99.90 percentile latency (ns)": 87900753, "Max latency (ns)": 148539955, "Mean latency (ns)": 15835616, "Min duration satisfied": "Yes", "Min latency (ns)": 14397372, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 63.02, "QPS w/o loadgen overhead": 63.15, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.474, "characteristics.good": 33737, "characteristics.samples_per_second": 63.15, "characteristics.samples_per_second.normalized_per_core": 15.7875, "characteristics.samples_per_second.normalized_per_processor": 63.15, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 62.5, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "6d1727644bcb9322", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 13668701, "90.00 percentile latency (ns)": 14394435, "90th percentile latency (ns)": 14394435, "95.00 percentile latency (ns)": 14557471, "97.00 percentile latency (ns)": 14717121, "99.00 percentile latency (ns)": 35523322, "99.90 percentile latency (ns)": 63682258, "Max latency (ns)": 98666412, "Mean latency (ns)": 14233795, "Min duration satisfied": "Yes", "Min latency (ns)": 12960931, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 70.11, "QPS w/o loadgen overhead": 70.26, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.24, "characteristics.good": 33620, "characteristics.samples_per_second": 70.26, "characteristics.samples_per_second.normalized_per_core": 17.565, "characteristics.samples_per_second.normalized_per_processor": 70.26, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "b694c61bff1fd552", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 26168277, "90.00 percentile latency (ns)": 26620914, "90th percentile latency (ns)": 26620914, "95.00 percentile latency (ns)": 27045092, "97.00 percentile latency (ns)": 30618763, "99.00 percentile latency (ns)": 52579949, "99.90 percentile latency (ns)": 89425149, "Max latency (ns)": 103197620, "Mean latency (ns)": 26880241, "Min duration satisfied": "Yes", "Min latency (ns)": 25486157, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 37.17, "QPS w/o loadgen overhead": 37.2, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.348, "characteristics.good": 32174, "characteristics.samples_per_second": 37.2, "characteristics.samples_per_second.normalized_per_core": 9.3, "characteristics.samples_per_second.normalized_per_processor": 37.2, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 35.7143, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "33bdc9e68b18d000", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 19227164, "90.00 percentile latency (ns)": 19637300, "90th percentile latency (ns)": 19637300, "95.00 percentile latency (ns)": 19874902, "97.00 percentile latency (ns)": 20611335, "99.00 percentile latency (ns)": 44813237, "99.90 percentile latency (ns)": 80380695, "Max latency (ns)": 108718551, "Mean latency (ns)": 19799138, "Min duration satisfied": "Yes", "Min latency (ns)": 18658056, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 50.43, "QPS w/o loadgen overhead": 50.51, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.246, "characteristics.good": 31623, "characteristics.samples_per_second": 50.51, "characteristics.samples_per_second.normalized_per_core": 12.6275, "characteristics.samples_per_second.normalized_per_processor": 50.51, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 47.619, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "6462693eda5f12a3", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 15245960, "90.00 percentile latency (ns)": 15653747, "90th percentile latency (ns)": 15653747, "95.00 percentile latency (ns)": 15992302, "97.00 percentile latency (ns)": 16236844, "99.00 percentile latency (ns)": 39295594, "99.90 percentile latency (ns)": 73318723, "Max latency (ns)": 138368876, "Mean latency (ns)": 15764963, "Min duration satisfied": "Yes", "Min latency (ns)": 14587256, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 63.34, "QPS w/o loadgen overhead": 63.43, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.668, "characteristics.good": 30834, "characteristics.samples_per_second": 63.43, "characteristics.samples_per_second.normalized_per_core": 15.8575, "characteristics.samples_per_second.normalized_per_processor": 63.43, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 62.5, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "51542eab496b3dce", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 26042473, "90.00 percentile latency (ns)": 26499408, "90th percentile latency (ns)": 26499408, "95.00 percentile latency (ns)": 26980581, "97.00 percentile latency (ns)": 32779201, "99.00 percentile latency (ns)": 53528211, "99.90 percentile latency (ns)": 97700102, "Max latency (ns)": 112756157, "Mean latency (ns)": 26826970, "Min duration satisfied": "Yes", "Min latency (ns)": 24819924, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 37.25, "QPS w/o loadgen overhead": 37.28, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.384, "characteristics.good": 33192, "characteristics.samples_per_second": 37.28, "characteristics.samples_per_second.normalized_per_core": 9.32, "characteristics.samples_per_second.normalized_per_processor": 37.28, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 34.4828, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "2ae53fa0481d885a", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3656164, "90.00 percentile latency (ns)": 4201463, "90th percentile latency (ns)": 4201463, "95.00 percentile latency (ns)": 4410809, "97.00 percentile latency (ns)": 4515673, "99.00 percentile latency (ns)": 4727734, "99.90 percentile latency (ns)": 36335888, "Max latency (ns)": 121167060, "Mean latency (ns)": 3854363, "Min duration satisfied": "Yes", "Min latency (ns)": 3065173, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 257.34, "QPS w/o loadgen overhead": 259.45, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.11, "characteristics.good": 24555, "characteristics.samples_per_second": 259.45, "characteristics.samples_per_second.normalized_per_core": 64.8625, "characteristics.samples_per_second.normalized_per_processor": 259.45, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "17f24d8c96317388", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7844752, "90.00 percentile latency (ns)": 8443447, "90th percentile latency (ns)": 8443447, "95.00 percentile latency (ns)": 8545668, "97.00 percentile latency (ns)": 8648277, "99.00 percentile latency (ns)": 11244444, "99.90 percentile latency (ns)": 42064909, "Max latency (ns)": 139771299, "Mean latency (ns)": 8185469, "Min duration satisfied": "Yes", "Min latency (ns)": 7140973, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 121.77, "QPS w/o loadgen overhead": 122.17, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 62.07, "characteristics.good": 31035, "characteristics.samples_per_second": 122.17, "characteristics.samples_per_second.normalized_per_core": 30.5425, "characteristics.samples_per_second.normalized_per_processor": 122.17, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "5661cba40af9587b", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 15713543, "90.00 percentile latency (ns)": 16392021, "90th percentile latency (ns)": 16392021, "95.00 percentile latency (ns)": 16643609, "97.00 percentile latency (ns)": 17042405, "99.00 percentile latency (ns)": 37723399, "99.90 percentile latency (ns)": 81375224, "Max latency (ns)": 139775124, "Mean latency (ns)": 16305375, "Min duration satisfied": "Yes", "Min latency (ns)": 14690904, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 61.23, "QPS w/o loadgen overhead": 61.33, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.224, "characteristics.good": 30612, "characteristics.samples_per_second": 61.33, "characteristics.samples_per_second.normalized_per_core": 15.3325, "characteristics.samples_per_second.normalized_per_processor": 61.33, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 58.8235, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "267f74ae0b18517a", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2152567, "90.00 percentile latency (ns)": 2850664, "90th percentile latency (ns)": 2850664, "95.00 percentile latency (ns)": 2952921, "97.00 percentile latency (ns)": 3047345, "99.00 percentile latency (ns)": 3228156, "99.90 percentile latency (ns)": 34381533, "Max latency (ns)": 81181014, "Mean latency (ns)": 2366757, "Min duration satisfied": "Yes", "Min latency (ns)": 1662725, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 416.57, "QPS w/o loadgen overhead": 422.52, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 42.384, "characteristics.good": 21192, "characteristics.samples_per_second": 422.52, "characteristics.samples_per_second.normalized_per_core": 105.63, "characteristics.samples_per_second.normalized_per_processor": 422.52, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "885df8f9487f411f", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 82086556, "90.00 percentile latency (ns)": 83343415, "90th percentile latency (ns)": 83343415, "95.00 percentile latency (ns)": 101159311, "97.00 percentile latency (ns)": 106704484, "99.00 percentile latency (ns)": 111438896, "99.90 percentile latency (ns)": 200731389, "Max latency (ns)": 226904541, "Mean latency (ns)": 84034952, "Min duration satisfied": "Yes", "Min latency (ns)": 80490824, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 11.9, "QPS w/o loadgen overhead": 11.9, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.012, "characteristics.good": 37506, "characteristics.samples_per_second": 11.9, "characteristics.samples_per_second.normalized_per_core": 2.975, "characteristics.samples_per_second.normalized_per_processor": 11.9, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.3-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.3-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.3_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 11.6279, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "464ab150eab57a8d", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10747063, "90.00 percentile latency (ns)": 11305860, "90th percentile latency (ns)": 11305860, "95.00 percentile latency (ns)": 11494677, "97.00 percentile latency (ns)": 11720615, "99.00 percentile latency (ns)": 23146178, "99.90 percentile latency (ns)": 43860714, "Max latency (ns)": 100204931, "Mean latency (ns)": 11096487, "Min duration satisfied": "Yes", "Min latency (ns)": 9900288, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 89.82, "QPS w/o loadgen overhead": 90.12, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.486, "characteristics.good": 32243, "characteristics.samples_per_second": 90.12, "characteristics.samples_per_second.normalized_per_core": 22.53, "characteristics.samples_per_second.normalized_per_processor": 90.12, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "701ceddb46a9bb58", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 128071312, "90.00 percentile latency (ns)": 139293186, "90th percentile latency (ns)": 139293186, "95.00 percentile latency (ns)": 150048960, "97.00 percentile latency (ns)": 152106359, "99.00 percentile latency (ns)": 157359306, "99.90 percentile latency (ns)": 272677015, "Max latency (ns)": 284267384, "Mean latency (ns)": 130864251, "Min duration satisfied": "Yes", "Min latency (ns)": 126188867, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.64, "QPS w/o loadgen overhead": 7.64, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 77.542, "characteristics.good": 38771, "characteristics.samples_per_second": 7.64, "characteristics.samples_per_second.normalized_per_core": 1.91, "characteristics.samples_per_second.normalized_per_processor": 7.64, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite2-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 7.35294, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "a9e2fb41048194e8", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4178899, "90.00 percentile latency (ns)": 4880912, "90th percentile latency (ns)": 4880912, "95.00 percentile latency (ns)": 5071806, "97.00 percentile latency (ns)": 5172673, "99.00 percentile latency (ns)": 5416338, "99.90 percentile latency (ns)": 37373800, "Max latency (ns)": 77531325, "Mean latency (ns)": 4448973, "Min duration satisfied": "Yes", "Min latency (ns)": 3767146, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 222.7, "QPS w/o loadgen overhead": 224.77, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.75, "characteristics.good": 28375, "characteristics.samples_per_second": 224.77, "characteristics.samples_per_second.normalized_per_core": 56.1925, "characteristics.samples_per_second.normalized_per_processor": 224.77, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "160a8849f4b6be96", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5400985, "90.00 percentile latency (ns)": 6066045, "90th percentile latency (ns)": 6066045, "95.00 percentile latency (ns)": 6195358, "97.00 percentile latency (ns)": 6266189, "99.00 percentile latency (ns)": 6645312, "99.90 percentile latency (ns)": 38567047, "Max latency (ns)": 130290540, "Mean latency (ns)": 5692032, "Min duration satisfied": "Yes", "Min latency (ns)": 4859292, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 174.31, "QPS w/o loadgen overhead": 175.68, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 57.002, "characteristics.good": 28501, "characteristics.samples_per_second": 175.68, "characteristics.samples_per_second.normalized_per_core": 43.92, "characteristics.samples_per_second.normalized_per_processor": 175.68, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "b8ebae4e5a407a83", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 88732418, "90.00 percentile latency (ns)": 90014897, "90th percentile latency (ns)": 90014897, "95.00 percentile latency (ns)": 112182507, "97.00 percentile latency (ns)": 114949333, "99.00 percentile latency (ns)": 118762262, "99.90 percentile latency (ns)": 216066724, "Max latency (ns)": 237982198, "Mean latency (ns)": 90949848, "Min duration satisfied": "Yes", "Min latency (ns)": 86974488, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 10.99, "QPS w/o loadgen overhead": 11.0, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.69, "characteristics.good": 37845, "characteristics.samples_per_second": 11.0, "characteristics.samples_per_second.normalized_per_core": 2.75, "characteristics.samples_per_second.normalized_per_processor": 11.0, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.4-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.4-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.4_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 10.6383, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "8dbca439bbddfb90", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6900583, "90.00 percentile latency (ns)": 7654810, "90th percentile latency (ns)": 7654810, "95.00 percentile latency (ns)": 7836954, "97.00 percentile latency (ns)": 7972060, "99.00 percentile latency (ns)": 10098180, "99.90 percentile latency (ns)": 41416479, "Max latency (ns)": 119594786, "Mean latency (ns)": 7220589, "Min duration satisfied": "Yes", "Min latency (ns)": 6193453, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 137.64, "QPS w/o loadgen overhead": 138.49, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 52.446, "characteristics.good": 26223, "characteristics.samples_per_second": 138.49, "characteristics.samples_per_second.normalized_per_core": 34.6225, "characteristics.samples_per_second.normalized_per_processor": 138.49, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "35691d4521b80bce", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 40034075, "90.00 percentile latency (ns)": 40643336, "90th percentile latency (ns)": 40643336, "95.00 percentile latency (ns)": 42310806, "97.00 percentile latency (ns)": 55493518, "99.00 percentile latency (ns)": 68292713, "99.90 percentile latency (ns)": 144549753, "Max latency (ns)": 177165584, "Mean latency (ns)": 41130886, "Min duration satisfied": "Yes", "Min latency (ns)": 39399309, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 24.3, "QPS w/o loadgen overhead": 24.31, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.724, "characteristics.good": 36862, "characteristics.samples_per_second": 24.31, "characteristics.samples_per_second.normalized_per_core": 6.0775, "characteristics.samples_per_second.normalized_per_processor": 24.31, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 22.7273, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "d342bea1f55d4d2a", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 30109897, "90.00 percentile latency (ns)": 30776339, "90th percentile latency (ns)": 30776339, "95.00 percentile latency (ns)": 31468039, "97.00 percentile latency (ns)": 40356827, "99.00 percentile latency (ns)": 58559720, "99.90 percentile latency (ns)": 102258722, "Max latency (ns)": 117605580, "Mean latency (ns)": 31036026, "Min duration satisfied": "Yes", "Min latency (ns)": 29034062, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 32.19, "QPS w/o loadgen overhead": 32.22, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.604, "characteristics.good": 33302, "characteristics.samples_per_second": 32.22, "characteristics.samples_per_second.normalized_per_core": 8.055, "characteristics.samples_per_second.normalized_per_processor": 32.22, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 30.303, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "1b1aa445228b13a1", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 41161187, "90.00 percentile latency (ns)": 42135232, "90th percentile latency (ns)": 42135232, "95.00 percentile latency (ns)": 46264747, "97.00 percentile latency (ns)": 55361231, "99.00 percentile latency (ns)": 66380324, "99.90 percentile latency (ns)": 117010824, "Max latency (ns)": 135704927, "Mean latency (ns)": 42208715, "Min duration satisfied": "Yes", "Min latency (ns)": 39866641, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 23.68, "QPS w/o loadgen overhead": 23.69, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.106, "characteristics.good": 34553, "characteristics.samples_per_second": 23.69, "characteristics.samples_per_second.normalized_per_core": 5.9225, "characteristics.samples_per_second.normalized_per_processor": 23.69, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.2558, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "c2b0291a433f8453", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11111142, "90.00 percentile latency (ns)": 11740089, "90th percentile latency (ns)": 11740089, "95.00 percentile latency (ns)": 11950656, "97.00 percentile latency (ns)": 12156816, "99.00 percentile latency (ns)": 25403378, "99.90 percentile latency (ns)": 49829056, "Max latency (ns)": 133821685, "Mean latency (ns)": 11576414, "Min duration satisfied": "Yes", "Min latency (ns)": 10265987, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 86.12, "QPS w/o loadgen overhead": 86.38, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.822, "characteristics.good": 29911, "characteristics.samples_per_second": 86.38, "characteristics.samples_per_second.normalized_per_core": 21.595, "characteristics.samples_per_second.normalized_per_processor": 86.38, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "ebbce6bb5381977b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20071378, "90.00 percentile latency (ns)": 20476027, "90th percentile latency (ns)": 20476027, "95.00 percentile latency (ns)": 20750456, "97.00 percentile latency (ns)": 22005336, "99.00 percentile latency (ns)": 47274006, "99.90 percentile latency (ns)": 98665285, "Max latency (ns)": 112908053, "Mean latency (ns)": 20681404, "Min duration satisfied": "Yes", "Min latency (ns)": 19354186, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 48.29, "QPS w/o loadgen overhead": 48.35, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.336, "characteristics.good": 34168, "characteristics.samples_per_second": 48.35, "characteristics.samples_per_second.normalized_per_core": 12.0875, "characteristics.samples_per_second.normalized_per_processor": 48.35, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 45.4545, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "9afd2a2d247ab8ea", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 33028614, "90.00 percentile latency (ns)": 33421635, "90th percentile latency (ns)": 33421635, "95.00 percentile latency (ns)": 34216803, "97.00 percentile latency (ns)": 43834235, "99.00 percentile latency (ns)": 60609376, "99.90 percentile latency (ns)": 131377652, "Max latency (ns)": 161012980, "Mean latency (ns)": 33962779, "Min duration satisfied": "Yes", "Min latency (ns)": 32468713, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 29.42, "QPS w/o loadgen overhead": 29.44, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.834, "characteristics.good": 34417, "characteristics.samples_per_second": 29.44, "characteristics.samples_per_second.normalized_per_core": 7.36, "characteristics.samples_per_second.normalized_per_processor": 29.44, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 27.7778, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "611c84176914f3f6", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 31451829, "90.00 percentile latency (ns)": 32083738, "90th percentile latency (ns)": 32083738, "95.00 percentile latency (ns)": 32430373, "97.00 percentile latency (ns)": 41180554, "99.00 percentile latency (ns)": 60013656, "99.90 percentile latency (ns)": 112404293, "Max latency (ns)": 161733637, "Mean latency (ns)": 32383627, "Min duration satisfied": "Yes", "Min latency (ns)": 30615635, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 30.85, "QPS w/o loadgen overhead": 30.88, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.19, "characteristics.good": 35595, "characteristics.samples_per_second": 30.88, "characteristics.samples_per_second.normalized_per_core": 7.72, "characteristics.samples_per_second.normalized_per_processor": 30.88, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite0-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 29.4118, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "a569a8240bcb8817", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 192227408, "90.00 percentile latency (ns)": 211456137, "90th percentile latency (ns)": 211456137, "95.00 percentile latency (ns)": 215048742, "97.00 percentile latency (ns)": 217075044, "99.00 percentile latency (ns)": 237112187, "99.90 percentile latency (ns)": 333504941, "Max latency (ns)": 343034144, "Mean latency (ns)": 196175451, "Min duration satisfied": "Yes", "Min latency (ns)": 190174694, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 5.1, "QPS w/o loadgen overhead": 5.1, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 79.268, "characteristics.good": 39634, "characteristics.samples_per_second": 5.1, "characteristics.samples_per_second.normalized_per_core": 1.275, "characteristics.samples_per_second.normalized_per_processor": 5.1, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite3-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 4.92611, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "4aa9b4474f007302", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 17684190, "90.00 percentile latency (ns)": 17964205, "90th percentile latency (ns)": 17964205, "95.00 percentile latency (ns)": 18214430, "97.00 percentile latency (ns)": 18624223, "99.00 percentile latency (ns)": 42920960, "99.90 percentile latency (ns)": 94354312, "Max latency (ns)": 115229144, "Mean latency (ns)": 18200962, "Min duration satisfied": "Yes", "Min latency (ns)": 17052978, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 54.87, "QPS w/o loadgen overhead": 54.94, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.216, "characteristics.good": 33108, "characteristics.samples_per_second": 54.94, "characteristics.samples_per_second.normalized_per_core": 13.735, "characteristics.samples_per_second.normalized_per_processor": 54.94, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 52.6316, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "3a7cc584c0c86431", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 95513637, "90.00 percentile latency (ns)": 97845640, "90th percentile latency (ns)": 97845640, "95.00 percentile latency (ns)": 118102153, "97.00 percentile latency (ns)": 120115314, "99.00 percentile latency (ns)": 124181094, "99.90 percentile latency (ns)": 235837196, "Max latency (ns)": 272873641, "Mean latency (ns)": 97798288, "Min duration satisfied": "Yes", "Min latency (ns)": 93930555, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 10.22, "QPS w/o loadgen overhead": 10.23, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.344, "characteristics.good": 38172, "characteristics.samples_per_second": 10.23, "characteristics.samples_per_second.normalized_per_core": 2.5575, "characteristics.samples_per_second.normalized_per_processor": 10.23, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite3-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 9.90099, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "309df0e4c58b553e", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5645941, "90.00 percentile latency (ns)": 6294034, "90th percentile latency (ns)": 6294034, "95.00 percentile latency (ns)": 6466004, "97.00 percentile latency (ns)": 6546258, "99.00 percentile latency (ns)": 6939588, "99.90 percentile latency (ns)": 39441827, "Max latency (ns)": 121397203, "Mean latency (ns)": 5931408, "Min duration satisfied": "Yes", "Min latency (ns)": 5063384, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 167.31, "QPS w/o loadgen overhead": 168.59, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 54.568, "characteristics.good": 27284, "characteristics.samples_per_second": 168.59, "characteristics.samples_per_second.normalized_per_core": 42.1475, "characteristics.samples_per_second.normalized_per_processor": 168.59, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "59bf450baa11afd9", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9622225, "90.00 percentile latency (ns)": 9879571, "90th percentile latency (ns)": 9879571, "95.00 percentile latency (ns)": 10020752, "97.00 percentile latency (ns)": 10214766, "99.00 percentile latency (ns)": 26496282, "99.90 percentile latency (ns)": 68882699, "Max latency (ns)": 82792572, "Mean latency (ns)": 10054290, "Min duration satisfied": "Yes", "Min latency (ns)": 9172751, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 99.27, "QPS w/o loadgen overhead": 99.46, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.478, "characteristics.good": 24739, "characteristics.samples_per_second": 99.46, "characteristics.samples_per_second.normalized_per_core": 24.865, "characteristics.samples_per_second.normalized_per_processor": 99.46, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "900f2784b3db234f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 18572484, "90.00 percentile latency (ns)": 19252682, "90th percentile latency (ns)": 19252682, "95.00 percentile latency (ns)": 19560003, "97.00 percentile latency (ns)": 20086015, "99.00 percentile latency (ns)": 44814149, "99.90 percentile latency (ns)": 85487903, "Max latency (ns)": 111258196, "Mean latency (ns)": 19222600, "Min duration satisfied": "Yes", "Min latency (ns)": 17786896, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 51.9, "QPS w/o loadgen overhead": 52.02, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.126, "characteristics.good": 34063, "characteristics.samples_per_second": 52.02, "characteristics.samples_per_second.normalized_per_core": 13.005, "characteristics.samples_per_second.normalized_per_processor": 52.02, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "4762c703f4bf1a85", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 52115418, "90.00 percentile latency (ns)": 52623891, "90th percentile latency (ns)": 52623891, "95.00 percentile latency (ns)": 59713531, "97.00 percentile latency (ns)": 74872415, "99.00 percentile latency (ns)": 80939794, "99.90 percentile latency (ns)": 161925628, "Max latency (ns)": 175032423, "Mean latency (ns)": 53488101, "Min duration satisfied": "Yes", "Min latency (ns)": 51156747, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 18.69, "QPS w/o loadgen overhead": 18.7, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.76, "characteristics.good": 37880, "characteristics.samples_per_second": 18.7, "characteristics.samples_per_second.normalized_per_core": 4.675, "characteristics.samples_per_second.normalized_per_processor": 18.7, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.8679, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "6ba3389bab4179c1", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 27521512, "90.00 percentile latency (ns)": 27926061, "90th percentile latency (ns)": 27926061, "95.00 percentile latency (ns)": 28374246, "97.00 percentile latency (ns)": 30554156, "99.00 percentile latency (ns)": 55295500, "99.90 percentile latency (ns)": 123514129, "Max latency (ns)": 147412670, "Mean latency (ns)": 28351997, "Min duration satisfied": "Yes", "Min latency (ns)": 26996248, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 35.25, "QPS w/o loadgen overhead": 35.27, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 65.75, "characteristics.good": 32875, "characteristics.samples_per_second": 35.27, "characteristics.samples_per_second.normalized_per_core": 8.8175, "characteristics.samples_per_second.normalized_per_processor": 35.27, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 33.3333, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "141a443aa20da162", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10706553, "90.00 percentile latency (ns)": 11322450, "90th percentile latency (ns)": 11322450, "95.00 percentile latency (ns)": 11466316, "97.00 percentile latency (ns)": 11606829, "99.00 percentile latency (ns)": 24456432, "99.90 percentile latency (ns)": 52585934, "Max latency (ns)": 96069165, "Mean latency (ns)": 11094303, "Min duration satisfied": "Yes", "Min latency (ns)": 9988121, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 89.83, "QPS w/o loadgen overhead": 90.14, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.768, "characteristics.good": 32384, "characteristics.samples_per_second": 90.14, "characteristics.samples_per_second.normalized_per_core": 22.535, "characteristics.samples_per_second.normalized_per_processor": 90.14, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "cc8cdfbd4dc2a626", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 43749182, "90.00 percentile latency (ns)": 44326201, "90th percentile latency (ns)": 44326201, "95.00 percentile latency (ns)": 47045334, "97.00 percentile latency (ns)": 61259847, "99.00 percentile latency (ns)": 73714323, "99.90 percentile latency (ns)": 149084630, "Max latency (ns)": 180312079, "Mean latency (ns)": 44957628, "Min duration satisfied": "Yes", "Min latency (ns)": 43010994, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 22.23, "QPS w/o loadgen overhead": 22.24, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.122, "characteristics.good": 35061, "characteristics.samples_per_second": 22.24, "characteristics.samples_per_second.normalized_per_core": 5.56, "characteristics.samples_per_second.normalized_per_processor": 22.24, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 20.8333, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "60d4d696e144b7c9", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10393800, "90.00 percentile latency (ns)": 11075466, "90th percentile latency (ns)": 11075466, "95.00 percentile latency (ns)": 11295614, "97.00 percentile latency (ns)": 11416016, "99.00 percentile latency (ns)": 22653234, "99.90 percentile latency (ns)": 52824850, "Max latency (ns)": 88912672, "Mean latency (ns)": 10804600, "Min duration satisfied": "Yes", "Min latency (ns)": 9706848, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 92.31, "QPS w/o loadgen overhead": 92.55, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.08, "characteristics.good": 32040, "characteristics.samples_per_second": 92.55, "characteristics.samples_per_second.normalized_per_core": 23.1375, "characteristics.samples_per_second.normalized_per_processor": 92.55, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "0c504cbfa967249b", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20547761, "90.00 percentile latency (ns)": 20855041, "90th percentile latency (ns)": 20855041, "95.00 percentile latency (ns)": 21052314, "97.00 percentile latency (ns)": 21662074, "99.00 percentile latency (ns)": 49719195, "99.90 percentile latency (ns)": 96935021, "Max latency (ns)": 160271968, "Mean latency (ns)": 21189448, "Min duration satisfied": "Yes", "Min latency (ns)": 19968947, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 47.14, "QPS w/o loadgen overhead": 47.19, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.282, "characteristics.good": 32141, "characteristics.samples_per_second": 47.19, "characteristics.samples_per_second.normalized_per_core": 11.7975, "characteristics.samples_per_second.normalized_per_processor": 47.19, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 45.4545, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "7b2ed533c7142588", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 41816715, "90.00 percentile latency (ns)": 42298497, "90th percentile latency (ns)": 42298497, "95.00 percentile latency (ns)": 44406956, "97.00 percentile latency (ns)": 60291571, "99.00 percentile latency (ns)": 72287741, "99.90 percentile latency (ns)": 147313691, "Max latency (ns)": 183488709, "Mean latency (ns)": 42992537, "Min duration satisfied": "Yes", "Min latency (ns)": 41057294, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 23.25, "QPS w/o loadgen overhead": 23.26, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.678, "characteristics.good": 36339, "characteristics.samples_per_second": 23.26, "characteristics.samples_per_second.normalized_per_core": 5.815, "characteristics.samples_per_second.normalized_per_processor": 23.26, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 22.2222, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "f9f6ff441b3b97b4", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11597588, "90.00 percentile latency (ns)": 12210520, "90th percentile latency (ns)": 12210520, "95.00 percentile latency (ns)": 12436729, "97.00 percentile latency (ns)": 12548845, "99.00 percentile latency (ns)": 28002087, "99.90 percentile latency (ns)": 49344680, "Max latency (ns)": 133243274, "Mean latency (ns)": 11995013, "Min duration satisfied": "Yes", "Min latency (ns)": 10723619, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 83.17, "QPS w/o loadgen overhead": 83.37, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.972, "characteristics.good": 32486, "characteristics.samples_per_second": 83.37, "characteristics.samples_per_second.normalized_per_core": 20.8425, "characteristics.samples_per_second.normalized_per_processor": 83.37, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "997440b8fe25c99b", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 55572980, "90.00 percentile latency (ns)": 56202380, "90th percentile latency (ns)": 56202380, "95.00 percentile latency (ns)": 66237486, "97.00 percentile latency (ns)": 80544763, "99.00 percentile latency (ns)": 85679996, "99.90 percentile latency (ns)": 167039778, "Max latency (ns)": 188782830, "Mean latency (ns)": 57094492, "Min duration satisfied": "Yes", "Min latency (ns)": 54471640, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.51, "QPS w/o loadgen overhead": 17.51, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.77, "characteristics.good": 35885, "characteristics.samples_per_second": 17.51, "characteristics.samples_per_second.normalized_per_core": 4.3775, "characteristics.samples_per_second.normalized_per_processor": 17.51, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 16.9492, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "ae12cab9855a4926", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 24482948, "90.00 percentile latency (ns)": 24856321, "90th percentile latency (ns)": 24856321, "95.00 percentile latency (ns)": 25279330, "97.00 percentile latency (ns)": 28023364, "99.00 percentile latency (ns)": 52943666, "99.90 percentile latency (ns)": 110063298, "Max latency (ns)": 156256434, "Mean latency (ns)": 25214687, "Min duration satisfied": "Yes", "Min latency (ns)": 23646478, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 39.63, "QPS w/o loadgen overhead": 39.66, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.75, "characteristics.good": 33375, "characteristics.samples_per_second": 39.66, "characteristics.samples_per_second.normalized_per_core": 9.915, "characteristics.samples_per_second.normalized_per_processor": 39.66, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 38.4615, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "23d895633466c6ef", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5654583, "90.00 percentile latency (ns)": 6291792, "90th percentile latency (ns)": 6291792, "95.00 percentile latency (ns)": 6442048, "97.00 percentile latency (ns)": 6571007, "99.00 percentile latency (ns)": 7630004, "99.90 percentile latency (ns)": 39619779, "Max latency (ns)": 121039902, "Mean latency (ns)": 5932313, "Min duration satisfied": "Yes", "Min latency (ns)": 5152261, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 167.29, "QPS w/o loadgen overhead": 168.57, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 47.14, "characteristics.good": 23570, "characteristics.samples_per_second": 168.57, "characteristics.samples_per_second.normalized_per_core": 42.1425, "characteristics.samples_per_second.normalized_per_processor": 168.57, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "bbaf249109636433", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11834592, "90.00 percentile latency (ns)": 12196489, "90th percentile latency (ns)": 12196489, "95.00 percentile latency (ns)": 12454343, "97.00 percentile latency (ns)": 12725329, "99.00 percentile latency (ns)": 27823745, "99.90 percentile latency (ns)": 50016437, "Max latency (ns)": 142495947, "Mean latency (ns)": 12257433, "Min duration satisfied": "Yes", "Min latency (ns)": 11154554, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 81.43, "QPS w/o loadgen overhead": 81.58, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.988, "characteristics.good": 28494, "characteristics.samples_per_second": 81.58, "characteristics.samples_per_second.normalized_per_core": 20.395, "characteristics.samples_per_second.normalized_per_processor": 81.58, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "2c824f4f8a108f31", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 18104919, "90.00 percentile latency (ns)": 18777635, "90th percentile latency (ns)": 18777635, "95.00 percentile latency (ns)": 19042974, "97.00 percentile latency (ns)": 20009230, "99.00 percentile latency (ns)": 44162000, "99.90 percentile latency (ns)": 79089605, "Max latency (ns)": 97511620, "Mean latency (ns)": 18738861, "Min duration satisfied": "Yes", "Min latency (ns)": 17204076, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 53.22, "QPS w/o loadgen overhead": 53.37, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.716, "characteristics.good": 31858, "characteristics.samples_per_second": 53.37, "characteristics.samples_per_second.normalized_per_core": 13.3425, "characteristics.samples_per_second.normalized_per_processor": 53.37, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 47.619, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "4b5fba00eefc0f3e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5175158, "90.00 percentile latency (ns)": 5782868, "90th percentile latency (ns)": 5782868, "95.00 percentile latency (ns)": 5997715, "97.00 percentile latency (ns)": 6112567, "99.00 percentile latency (ns)": 6439097, "99.90 percentile latency (ns)": 38650917, "Max latency (ns)": 115610075, "Mean latency (ns)": 5444768, "Min duration satisfied": "Yes", "Min latency (ns)": 4635668, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 182.46, "QPS w/o loadgen overhead": 183.66, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 55.54, "characteristics.good": 27770, "characteristics.samples_per_second": 183.66, "characteristics.samples_per_second.normalized_per_core": 45.915, "characteristics.samples_per_second.normalized_per_processor": 183.66, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "3f4aaedbfb354faa", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 16638030, "90.00 percentile latency (ns)": 16981933, "90th percentile latency (ns)": 16981933, "95.00 percentile latency (ns)": 17238576, "97.00 percentile latency (ns)": 17535106, "99.00 percentile latency (ns)": 42544779, "99.90 percentile latency (ns)": 95881346, "Max latency (ns)": 148072032, "Mean latency (ns)": 17179475, "Min duration satisfied": "Yes", "Min latency (ns)": 16088412, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 58.13, "QPS w/o loadgen overhead": 58.21, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.322, "characteristics.good": 29661, "characteristics.samples_per_second": 58.21, "characteristics.samples_per_second.normalized_per_core": 14.5525, "characteristics.samples_per_second.normalized_per_processor": 58.21, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 58.8235, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "d428f454a58ba63c", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 407943838, "90.00 percentile latency (ns)": 492011821, "90th percentile latency (ns)": 492011821, "95.00 percentile latency (ns)": 517999290, "97.00 percentile latency (ns)": 526708070, "99.00 percentile latency (ns)": 542772414, "99.90 percentile latency (ns)": 561651309, "Max latency (ns)": 567638809, "Mean latency (ns)": 401206749, "Min duration satisfied": "Yes", "Min latency (ns)": 309164543, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.49, "QPS w/o loadgen overhead": 2.49, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.power": 7.821816666666669, "characteristics.power.normalized_per_core": 0.9777270833333336, "characteristics.power.normalized_per_processor": 7.821816666666669, "characteristics.samples_per_second": 2.49, "characteristics.samples_per_second.normalized_per_core": 0.31125, "characteristics.samples_per_second.normalized_per_processor": 2.49, "characteristics.total": 50000, "ck_system": "xavier-power.2-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MODE_15W", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/results/xavier-power.2-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.2-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.5, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "6be95c4a2581d530", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite" }, { "50.00 percentile latency (ns)": 81749329, "90.00 percentile latency (ns)": 86848938, "90th percentile latency (ns)": 86848938, "95.00 percentile latency (ns)": 90958179, "97.00 percentile latency (ns)": 95179347, "99.00 percentile latency (ns)": 101553741, "99.90 percentile latency (ns)": 168817721, "Max latency (ns)": 228939300, "Mean latency (ns)": 81525886, "Min duration satisfied": "Yes", "Min latency (ns)": 73707718, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 12.26, "QPS w/o loadgen overhead": 12.27, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.power": 27.825416666666673, "characteristics.power.normalized_per_core": 3.478177083333334, "characteristics.power.normalized_per_processor": 27.825416666666673, "characteristics.samples_per_second": 12.27, "characteristics.samples_per_second.normalized_per_core": 1.53375, "characteristics.samples_per_second.normalized_per_processor": 12.27, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/results/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.5, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "f72786d4dc034298", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite" }, { "50.00 percentile latency (ns)": 157467016, "90.00 percentile latency (ns)": 251867040, "90th percentile latency (ns)": 251867040, "95.00 percentile latency (ns)": 256426334, "97.00 percentile latency (ns)": 259289467, "99.00 percentile latency (ns)": 267009722, "99.90 percentile latency (ns)": 276770992, "Max latency (ns)": 304786208, "Mean latency (ns)": 179884537, "Min duration satisfied": "Yes", "Min latency (ns)": 129398052, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 5.56, "QPS w/o loadgen overhead": 5.56, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.power": 8.418358333333329, "characteristics.power.normalized_per_core": 1.052294791666666, "characteristics.power.normalized_per_processor": 8.418358333333329, "characteristics.samples_per_second": 5.56, "characteristics.samples_per_second.normalized_per_core": 0.695, "characteristics.samples_per_second.normalized_per_processor": 5.56, "characteristics.total": 50000, "ck_system": "xavier-power.3-fan.cool-armnn-v21.05-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.05 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MODE_30W_ALL", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/results/xavier-power.3-fan.cool-armnn-v21.05-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.3-fan.cool-armnn-v21.05-neon", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.33333, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "443683070d62dbf0", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 156584585, "90.00 percentile latency (ns)": 259068378, "90th percentile latency (ns)": 259068378, "95.00 percentile latency (ns)": 264106053, "97.00 percentile latency (ns)": 266863766, "99.00 percentile latency (ns)": 273830197, "99.90 percentile latency (ns)": 297270047, "Max latency (ns)": 321363778, "Mean latency (ns)": 182928268, "Min duration satisfied": "Yes", "Min latency (ns)": 138950099, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 5.46, "QPS w/o loadgen overhead": 5.47, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.power": 8.86999333333334, "characteristics.power.normalized_per_core": 1.1087491666666676, "characteristics.power.normalized_per_processor": 8.86999333333334, "characteristics.samples_per_second": 5.47, "characteristics.samples_per_second.normalized_per_core": 0.68375, "characteristics.samples_per_second.normalized_per_processor": 5.47, "characteristics.total": 50000, "ck_system": "xavier-power.4-fan.cool-armnn-v21.05-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.05 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MODE_30W_6CORE", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/results/xavier-power.4-fan.cool-armnn-v21.05-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.4-fan.cool-armnn-v21.05-neon", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.33333, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "738b6257afbd2da0", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 57554560, "90.00 percentile latency (ns)": 72084529, "90th percentile latency (ns)": 72084529, "95.00 percentile latency (ns)": 77972086, "97.00 percentile latency (ns)": 82334804, "99.00 percentile latency (ns)": 96149085, "99.90 percentile latency (ns)": 108694817, "Max latency (ns)": 117254487, "Mean latency (ns)": 60072796, "Min duration satisfied": "Yes", "Min latency (ns)": 50053064, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.64, "QPS w/o loadgen overhead": 16.65, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.power": 22.208608333333345, "characteristics.power.normalized_per_core": 2.776076041666668, "characteristics.power.normalized_per_processor": 22.208608333333345, "characteristics.samples_per_second": 16.65, "characteristics.samples_per_second.normalized_per_core": 2.08125, "characteristics.samples_per_second.normalized_per_processor": 16.65, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.05-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.05 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/results/xavier-power.0-fan.cool-armnn-v21.05-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.05-neon", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.5, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "d851d6fc141a88b5", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 152211465, "90.00 percentile latency (ns)": 212559896, "90th percentile latency (ns)": 212559896, "95.00 percentile latency (ns)": 219302600, "97.00 percentile latency (ns)": 222617713, "99.00 percentile latency (ns)": 230715347, "99.90 percentile latency (ns)": 245439769, "Max latency (ns)": 249286150, "Mean latency (ns)": 166880905, "Min duration satisfied": "Yes", "Min latency (ns)": 127849877, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 5.99, "QPS w/o loadgen overhead": 5.99, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.power": 9.850795341098172, "characteristics.power.normalized_per_core": 1.2313494176372715, "characteristics.power.normalized_per_processor": 9.850795341098172, "characteristics.samples_per_second": 5.99, "characteristics.samples_per_second.normalized_per_core": 0.74875, "characteristics.samples_per_second.normalized_per_processor": 5.99, "characteristics.total": 50000, "ck_system": "xavier-power.5-fan.cool-armnn-v21.05-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.05 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MODE_30W_4CORE", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/results/xavier-power.5-fan.cool-armnn-v21.05-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.5-fan.cool-armnn-v21.05-neon", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.33333, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "229a929c04133887", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 460391961, "90.00 percentile latency (ns)": 513365769, "90th percentile latency (ns)": 513365769, "95.00 percentile latency (ns)": 540086550, "97.00 percentile latency (ns)": 566083793, "99.00 percentile latency (ns)": 599433317, "99.90 percentile latency (ns)": 629419677, "Max latency (ns)": 636100157, "Mean latency (ns)": 468588937, "Min duration satisfied": "Yes", "Min latency (ns)": 410442585, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.13, "QPS w/o loadgen overhead": 2.13, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.samples_per_second": 2.13, "characteristics.samples_per_second.normalized_per_core": 0.355, "characteristics.samples_per_second.normalized_per_processor": 2.13, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "4435c976bfa3d92e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite" }, { "50.00 percentile latency (ns)": 297026104, "90.00 percentile latency (ns)": 312576717, "90th percentile latency (ns)": 312576717, "95.00 percentile latency (ns)": 314930190, "97.00 percentile latency (ns)": 316858971, "99.00 percentile latency (ns)": 353617842, "99.90 percentile latency (ns)": 400217240, "Max latency (ns)": 409538342, "Mean latency (ns)": 301115888, "Min duration satisfied": "Yes", "Min latency (ns)": 292298589, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 3.32, "QPS w/o loadgen overhead": 3.32, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.samples_per_second": 3.32, "characteristics.samples_per_second.normalized_per_core": 0.83, "characteristics.samples_per_second.normalized_per_processor": 3.32, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.off-armnn-v21.05-neon", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan off", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.05 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/results/rpi4coral-fan.off-armnn-v21.05-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.off-armnn-v21.05-neon", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.33333, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "aae77fae4bd7a617", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 261222673, "90.00 percentile latency (ns)": 295216100, "90th percentile latency (ns)": 295216100, "95.00 percentile latency (ns)": 302924911, "97.00 percentile latency (ns)": 309871259, "99.00 percentile latency (ns)": 318383207, "99.90 percentile latency (ns)": 341641653, "Max latency (ns)": 371321129, "Mean latency (ns)": 253421870, "Min duration satisfied": "Yes", "Min latency (ns)": 205252846, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 3.95, "QPS w/o loadgen overhead": 3.95, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.power": 9.124064891846928, "characteristics.power.normalized_per_core": 1.140508111480866, "characteristics.power.normalized_per_processor": 9.124064891846928, "characteristics.samples_per_second": 3.95, "characteristics.samples_per_second.normalized_per_core": 0.49375, "characteristics.samples_per_second.normalized_per_processor": 3.95, "characteristics.total": 50000, "ck_system": "xavier-power.4-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MODE_30W_6CORE", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/results/xavier-power.4-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.4-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.33333, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "a295b228aa5095c5", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite" }, { "50.00 percentile latency (ns)": 536148159, "90.00 percentile latency (ns)": 552886733, "90th percentile latency (ns)": 552886733, "95.00 percentile latency (ns)": 557881108, "97.00 percentile latency (ns)": 573400901, "99.00 percentile latency (ns)": 652675954, "99.90 percentile latency (ns)": 797214609, "Max latency (ns)": 827329240, "Mean latency (ns)": 539672784, "Min duration satisfied": "Yes", "Min latency (ns)": 518672689, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.85, "QPS w/o loadgen overhead": 1.85, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.samples_per_second": 1.85, "characteristics.samples_per_second.normalized_per_core": 0.4625, "characteristics.samples_per_second.normalized_per_processor": 1.85, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.off-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan off", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/results/rpi4coral-fan.off-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.off-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "b9087348b098b448", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite" }, { "50.00 percentile latency (ns)": 246667258, "90.00 percentile latency (ns)": 377539337, "90th percentile latency (ns)": 377539337, "95.00 percentile latency (ns)": 384879527, "97.00 percentile latency (ns)": 390054264, "99.00 percentile latency (ns)": 396472655, "99.90 percentile latency (ns)": 401502604, "Max latency (ns)": 403894558, "Mean latency (ns)": 278242763, "Min duration satisfied": "Yes", "Min latency (ns)": 205223095, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 3.59, "QPS w/o loadgen overhead": 3.59, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.power": 7.587993344425955, "characteristics.power.normalized_per_core": 0.9484991680532444, "characteristics.power.normalized_per_processor": 7.587993344425955, "characteristics.samples_per_second": 3.59, "characteristics.samples_per_second.normalized_per_core": 0.44875, "characteristics.samples_per_second.normalized_per_processor": 3.59, "characteristics.total": 50000, "ck_system": "xavier-power.2-fan.cool-armnn-v21.05-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.05 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MODE_15W", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/results/xavier-power.2-fan.cool-armnn-v21.05-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.2-fan.cool-armnn-v21.05-neon", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.33333, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "34b9981aa9c4cfca", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 295843908, "90.00 percentile latency (ns)": 311639597, "90th percentile latency (ns)": 311639597, "95.00 percentile latency (ns)": 314241209, "97.00 percentile latency (ns)": 316654917, "99.00 percentile latency (ns)": 378801909, "99.90 percentile latency (ns)": 440793201, "Max latency (ns)": 440983308, "Mean latency (ns)": 300423447, "Min duration satisfied": "Yes", "Min latency (ns)": 291046324, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 3.33, "QPS w/o loadgen overhead": 3.33, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.samples_per_second": 3.33, "characteristics.samples_per_second.normalized_per_core": 0.8325, "characteristics.samples_per_second.normalized_per_processor": 3.33, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.05-neon", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.05 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/results/rpi4coral-fan.on-armnn-v21.05-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.05-neon", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.33333, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "84ae990f80749514", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 232293726, "90.00 percentile latency (ns)": 300953697, "90th percentile latency (ns)": 300953697, "95.00 percentile latency (ns)": 315414475, "97.00 percentile latency (ns)": 321732839, "99.00 percentile latency (ns)": 336250947, "99.90 percentile latency (ns)": 359908564, "Max latency (ns)": 362011346, "Mean latency (ns)": 240223220, "Min duration satisfied": "Yes", "Min latency (ns)": 184048192, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 4.16, "QPS w/o loadgen overhead": 4.16, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.power": 9.645457570715468, "characteristics.power.normalized_per_core": 1.2056821963394335, "characteristics.power.normalized_per_processor": 9.645457570715468, "characteristics.samples_per_second": 4.16, "characteristics.samples_per_second.normalized_per_core": 0.52, "characteristics.samples_per_second.normalized_per_processor": 4.16, "characteristics.total": 50000, "ck_system": "xavier-power.3-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MODE_30W_ALL", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/results/xavier-power.3-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.3-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.33333, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "4f4717888ecd64a4", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite" }, { "50.00 percentile latency (ns)": 522491675, "90.00 percentile latency (ns)": 537899368, "90th percentile latency (ns)": 537899368, "95.00 percentile latency (ns)": 541278241, "97.00 percentile latency (ns)": 544734426, "99.00 percentile latency (ns)": 565368105, "99.90 percentile latency (ns)": 647591910, "Max latency (ns)": 651126173, "Mean latency (ns)": 527419333, "Min duration satisfied": "Yes", "Min latency (ns)": 516095950, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.9, "QPS w/o loadgen overhead": 1.9, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.samples_per_second": 1.9, "characteristics.samples_per_second.normalized_per_core": 0.475, "characteristics.samples_per_second.normalized_per_processor": 1.9, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "http://raspberrypiwiki.com/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/results/rpi4coral-fan.on-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.5.0-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "1ce375a02408b854", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite" }, { "50.00 percentile latency (ns)": 236424870, "90.00 percentile latency (ns)": 264464129, "90th percentile latency (ns)": 264464129, "95.00 percentile latency (ns)": 271275991, "97.00 percentile latency (ns)": 276331724, "99.00 percentile latency (ns)": 287053312, "99.90 percentile latency (ns)": 302183430, "Max latency (ns)": 303749442, "Mean latency (ns)": 231711676, "Min duration satisfied": "Yes", "Min latency (ns)": 196787152, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 4.31, "QPS w/o loadgen overhead": 4.32, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.power": 10.77809816971715, "characteristics.power.normalized_per_core": 1.3472622712146438, "characteristics.power.normalized_per_processor": 10.77809816971715, "characteristics.samples_per_second": 4.32, "characteristics.samples_per_second.normalized_per_core": 0.54, "characteristics.samples_per_second.normalized_per_processor": 4.32, "characteristics.total": 50000, "ck_system": "xavier-power.5-fan.cool-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MODE_30W_4CORE", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/Krai/results/xavier-power.5-fan.cool-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.5", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.5-fan.cool-tflite-v2.5.0-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.33333, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "82a9df9e33940819", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite" }, { "50.00 percentile latency (ns)": 26927247, "90.00 percentile latency (ns)": 31318871, "90th percentile latency (ns)": 31318871, "95.00 percentile latency (ns)": 36899930, "97.00 percentile latency (ns)": 37587758, "99.00 percentile latency (ns)": 39058015, "99.90 percentile latency (ns)": 44301556, "Max latency (ns)": 51836597, "Mean latency (ns)": 27981780, "Min duration satisfied": "Yes", "Min latency (ns)": 26396996, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 35.72, "QPS w/o loadgen overhead": 35.74, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.242, "characteristics.good": 34121, "characteristics.power": 6.641099999999998, "characteristics.power.normalized_per_core": 0.05534249999999998, "characteristics.power.normalized_per_processor": 6.641099999999998, "characteristics.samples_per_second": 35.74, "characteristics.samples_per_second.normalized_per_core": 0.29783333333333334, "characteristics.samples_per_second.normalized_per_processor": 35.74, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 32.2581, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "3b59247ecb486fa5", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 37627723, "90.00 percentile latency (ns)": 46434374, "90th percentile latency (ns)": 46434374, "95.00 percentile latency (ns)": 52318768, "97.00 percentile latency (ns)": 53135478, "99.00 percentile latency (ns)": 55435496, "99.90 percentile latency (ns)": 61709697, "Max latency (ns)": 70013998, "Mean latency (ns)": 39312984, "Min duration satisfied": "Yes", "Min latency (ns)": 36530434, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 25.43, "QPS w/o loadgen overhead": 25.44, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.738, "characteristics.good": 35369, "characteristics.power": 6.2585000000000175, "characteristics.power.normalized_per_core": 0.052154166666666814, "characteristics.power.normalized_per_processor": 6.2585000000000175, "characteristics.samples_per_second": 25.44, "characteristics.samples_per_second.normalized_per_core": 0.21200000000000002, "characteristics.samples_per_second.normalized_per_processor": 25.44, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 22.7273, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "5b48e580f30b4572", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 498359029, "90.00 percentile latency (ns)": 516635786, "90th percentile latency (ns)": 516635786, "95.00 percentile latency (ns)": 564303750, "97.00 percentile latency (ns)": 625818939, "99.00 percentile latency (ns)": 643976624, "99.90 percentile latency (ns)": 654377556, "Max latency (ns)": 665401222, "Mean latency (ns)": 503697151, "Min duration satisfied": "Yes", "Min latency (ns)": 482237105, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.99, "QPS w/o loadgen overhead": 1.99, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 78.564, "characteristics.good": 39282, "characteristics.power": 7.229783693843598, "characteristics.power.normalized_per_core": 0.06024819744869665, "characteristics.power.normalized_per_processor": 7.229783693843598, "characteristics.samples_per_second": 1.99, "characteristics.samples_per_second.normalized_per_core": 0.016583333333333332, "characteristics.samples_per_second.normalized_per_processor": 1.99, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite4-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 1.99601, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "3b18d0a3446e7a92", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 14891023, "90.00 percentile latency (ns)": 18782367, "90th percentile latency (ns)": 18782367, "95.00 percentile latency (ns)": 22197527, "97.00 percentile latency (ns)": 23125107, "99.00 percentile latency (ns)": 24320693, "99.90 percentile latency (ns)": 28601603, "Max latency (ns)": 35321197, "Mean latency (ns)": 15878773, "Min duration satisfied": "Yes", "Min latency (ns)": 14269250, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 62.91, "QPS w/o loadgen overhead": 62.98, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.65, "characteristics.good": 31825, "characteristics.power": 6.395249999999999, "characteristics.power.normalized_per_core": 0.053293749999999994, "characteristics.power.normalized_per_processor": 6.395249999999999, "characteristics.samples_per_second": 62.98, "characteristics.samples_per_second.normalized_per_core": 0.5248333333333333, "characteristics.samples_per_second.normalized_per_processor": 62.98, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 62.5, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "9805955381fb6867", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 8749586, "90.00 percentile latency (ns)": 10462162, "90th percentile latency (ns)": 10462162, "95.00 percentile latency (ns)": 11151223, "97.00 percentile latency (ns)": 11377209, "99.00 percentile latency (ns)": 12189591, "99.90 percentile latency (ns)": 15189910, "Max latency (ns)": 31910061, "Mean latency (ns)": 9131376, "Min duration satisfied": "Yes", "Min latency (ns)": 8402161, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 109.35, "QPS w/o loadgen overhead": 109.51, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 52.172, "characteristics.good": 26086, "characteristics.power": 5.68453333333333, "characteristics.power.normalized_per_core": 0.04737111111111108, "characteristics.power.normalized_per_processor": 5.68453333333333, "characteristics.samples_per_second": 109.51, "characteristics.samples_per_second.normalized_per_core": 0.9125833333333334, "characteristics.samples_per_second.normalized_per_processor": 109.51, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "9912ce6799c03498", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 10102440, "90.00 percentile latency (ns)": 12444369, "90th percentile latency (ns)": 12444369, "95.00 percentile latency (ns)": 13431346, "97.00 percentile latency (ns)": 13768284, "99.00 percentile latency (ns)": 14728799, "99.90 percentile latency (ns)": 17806397, "Max latency (ns)": 25568945, "Mean latency (ns)": 10511273, "Min duration satisfied": "Yes", "Min latency (ns)": 9782114, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 95.01, "QPS w/o loadgen overhead": 95.14, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.402, "characteristics.good": 29201, "characteristics.power": 5.849033333333329, "characteristics.power.normalized_per_core": 0.048741944444444414, "characteristics.power.normalized_per_processor": 5.849033333333329, "characteristics.samples_per_second": 95.14, "characteristics.samples_per_second.normalized_per_core": 0.7928333333333334, "characteristics.samples_per_second.normalized_per_processor": 95.14, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "969854094c05bcb3", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 20505581, "90.00 percentile latency (ns)": 23797225, "90th percentile latency (ns)": 23797225, "95.00 percentile latency (ns)": 25556720, "97.00 percentile latency (ns)": 26072857, "99.00 percentile latency (ns)": 27351823, "99.90 percentile latency (ns)": 31341045, "Max latency (ns)": 43238571, "Mean latency (ns)": 21171119, "Min duration satisfied": "Yes", "Min latency (ns)": 19918969, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 47.2, "QPS w/o loadgen overhead": 47.23, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 60.976, "characteristics.good": 30488, "characteristics.power": 5.777050000000016, "characteristics.power.normalized_per_core": 0.04814208333333347, "characteristics.power.normalized_per_processor": 5.777050000000016, "characteristics.samples_per_second": 47.23, "characteristics.samples_per_second.normalized_per_core": 0.3935833333333333, "characteristics.samples_per_second.normalized_per_processor": 47.23, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 41.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c54b9223c9cb6817", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 5289071, "90.00 percentile latency (ns)": 7157948, "90th percentile latency (ns)": 7157948, "95.00 percentile latency (ns)": 7497352, "97.00 percentile latency (ns)": 7691836, "99.00 percentile latency (ns)": 8236642, "99.90 percentile latency (ns)": 10976871, "Max latency (ns)": 25998535, "Mean latency (ns)": 5588792, "Min duration satisfied": "Yes", "Min latency (ns)": 4845589, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 178.42, "QPS w/o loadgen overhead": 178.93, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 48.114, "characteristics.good": 24057, "characteristics.power": 6.097333333333326, "characteristics.power.normalized_per_core": 0.05081111111111105, "characteristics.power.normalized_per_processor": 6.097333333333326, "characteristics.samples_per_second": 178.93, "characteristics.samples_per_second.normalized_per_core": 1.4910833333333333, "characteristics.samples_per_second.normalized_per_processor": 178.93, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "42c9c734262ae5e1", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 59144569, "90.00 percentile latency (ns)": 71464601, "90th percentile latency (ns)": 71464601, "95.00 percentile latency (ns)": 76529591, "97.00 percentile latency (ns)": 85739377, "99.00 percentile latency (ns)": 88612379, "99.90 percentile latency (ns)": 94565119, "Max latency (ns)": 97195341, "Mean latency (ns)": 61649435, "Min duration satisfied": "Yes", "Min latency (ns)": 57689219, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.22, "QPS w/o loadgen overhead": 16.22, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.068, "characteristics.good": 37534, "characteristics.power": 6.399833333333327, "characteristics.power.normalized_per_core": 0.0533319444444444, "characteristics.power.normalized_per_processor": 6.399833333333327, "characteristics.samples_per_second": 16.22, "characteristics.samples_per_second.normalized_per_core": 0.13516666666666666, "characteristics.samples_per_second.normalized_per_processor": 16.22, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite0-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.873, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "fba6ddf1f6f96377", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 6636900, "90.00 percentile latency (ns)": 9143266, "90th percentile latency (ns)": 9143266, "95.00 percentile latency (ns)": 9517443, "97.00 percentile latency (ns)": 9755771, "99.00 percentile latency (ns)": 10361053, "99.90 percentile latency (ns)": 13243985, "Max latency (ns)": 29047484, "Mean latency (ns)": 7112955, "Min duration satisfied": "Yes", "Min latency (ns)": 6118948, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 140.28, "QPS w/o loadgen overhead": 140.59, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 50.164, "characteristics.good": 25082, "characteristics.power": 6.145983333333329, "characteristics.power.normalized_per_core": 0.05121652777777774, "characteristics.power.normalized_per_processor": 6.145983333333329, "characteristics.samples_per_second": 140.59, "characteristics.samples_per_second.normalized_per_core": 1.1715833333333334, "characteristics.samples_per_second.normalized_per_processor": 140.59, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ef1bec8d4dce6653", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 12020698, "90.00 percentile latency (ns)": 16047213, "90th percentile latency (ns)": 16047213, "95.00 percentile latency (ns)": 17045701, "97.00 percentile latency (ns)": 17494015, "99.00 percentile latency (ns)": 18502404, "99.90 percentile latency (ns)": 22119244, "Max latency (ns)": 38053573, "Mean latency (ns)": 12657154, "Min duration satisfied": "Yes", "Min latency (ns)": 11420028, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 78.91, "QPS w/o loadgen overhead": 79.01, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 60.842, "characteristics.good": 30421, "characteristics.power": 6.149750000000004, "characteristics.power.normalized_per_core": 0.0512479166666667, "characteristics.power.normalized_per_processor": 6.149750000000004, "characteristics.samples_per_second": 79.01, "characteristics.samples_per_second.normalized_per_core": 0.6584166666666668, "characteristics.samples_per_second.normalized_per_processor": 79.01, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "3a0487e41389aa89", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 27327896, "90.00 percentile latency (ns)": 33920720, "90th percentile latency (ns)": 33920720, "95.00 percentile latency (ns)": 38656213, "97.00 percentile latency (ns)": 39449292, "99.00 percentile latency (ns)": 41234345, "99.90 percentile latency (ns)": 47924800, "Max latency (ns)": 58759200, "Mean latency (ns)": 28718924, "Min duration satisfied": "Yes", "Min latency (ns)": 26710854, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 34.8, "QPS w/o loadgen overhead": 34.82, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.016, "characteristics.good": 34508, "characteristics.power": 6.24600000000001, "characteristics.power.normalized_per_core": 0.05205000000000008, "characteristics.power.normalized_per_processor": 6.24600000000001, "characteristics.samples_per_second": 34.82, "characteristics.samples_per_second.normalized_per_core": 0.2901666666666667, "characteristics.samples_per_second.normalized_per_processor": 34.82, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 30.303, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c845464eb79183e6", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 152988394, "90.00 percentile latency (ns)": 170100605, "90th percentile latency (ns)": 170100605, "95.00 percentile latency (ns)": 175787610, "97.00 percentile latency (ns)": 186616807, "99.00 percentile latency (ns)": 250991789, "99.90 percentile latency (ns)": 287867102, "Max latency (ns)": 293596238, "Mean latency (ns)": 159485998, "Min duration satisfied": "Yes", "Min latency (ns)": 151451151, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 6.27, "QPS w/o loadgen overhead": 6.27, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.116, "characteristics.good": 37058, "characteristics.power": 7.22601666666668, "characteristics.power.normalized_per_core": 0.06021680555555566, "characteristics.power.normalized_per_processor": 7.22601666666668, "characteristics.samples_per_second": 6.27, "characteristics.samples_per_second.normalized_per_core": 0.05225, "characteristics.samples_per_second.normalized_per_processor": 6.27, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite1-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 6.32911, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "87e4ba68f79895a2", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 11709220, "90.00 percentile latency (ns)": 16882439, "90th percentile latency (ns)": 16882439, "95.00 percentile latency (ns)": 17419722, "97.00 percentile latency (ns)": 17932100, "99.00 percentile latency (ns)": 18979187, "99.90 percentile latency (ns)": 22676605, "Max latency (ns)": 37461848, "Mean latency (ns)": 12701595, "Min duration satisfied": "Yes", "Min latency (ns)": 10973866, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 78.63, "QPS w/o loadgen overhead": 78.73, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.948, "characteristics.good": 30974, "characteristics.power": 6.262183333333336, "characteristics.power.normalized_per_core": 0.05218486111111113, "characteristics.power.normalized_per_processor": 6.262183333333336, "characteristics.samples_per_second": 78.73, "characteristics.samples_per_second.normalized_per_core": 0.6560833333333334, "characteristics.samples_per_second.normalized_per_processor": 78.73, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "b2b9fc3631f922b4", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 12597697, "90.00 percentile latency (ns)": 16986375, "90th percentile latency (ns)": 16986375, "95.00 percentile latency (ns)": 19608564, "97.00 percentile latency (ns)": 20369396, "99.00 percentile latency (ns)": 21377480, "99.90 percentile latency (ns)": 24588830, "Max latency (ns)": 39851963, "Mean latency (ns)": 13446964, "Min duration satisfied": "Yes", "Min latency (ns)": 12000483, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 74.27, "QPS w/o loadgen overhead": 74.37, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 62.264, "characteristics.good": 31132, "characteristics.power": 6.628483333333332, "characteristics.power.normalized_per_core": 0.0552373611111111, "characteristics.power.normalized_per_processor": 6.628483333333332, "characteristics.samples_per_second": 74.37, "characteristics.samples_per_second.normalized_per_core": 0.61975, "characteristics.samples_per_second.normalized_per_processor": 74.37, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "5c45fc84194852ef", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 6618888, "90.00 percentile latency (ns)": 9243242, "90th percentile latency (ns)": 9243242, "95.00 percentile latency (ns)": 9492255, "97.00 percentile latency (ns)": 9706383, "99.00 percentile latency (ns)": 10525902, "99.90 percentile latency (ns)": 13955963, "Max latency (ns)": 34805199, "Mean latency (ns)": 7100938, "Min duration satisfied": "Yes", "Min latency (ns)": 5849422, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 140.48, "QPS w/o loadgen overhead": 140.83, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.19, "characteristics.good": 24595, "characteristics.power": 6.0075166666666435, "characteristics.power.normalized_per_core": 0.0500626388888887, "characteristics.power.normalized_per_processor": 6.0075166666666435, "characteristics.samples_per_second": 140.83, "characteristics.samples_per_second.normalized_per_core": 1.1735833333333334, "characteristics.samples_per_second.normalized_per_processor": 140.83, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "529224738a8484a9", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 43137508, "90.00 percentile latency (ns)": 53278250, "90th percentile latency (ns)": 53278250, "95.00 percentile latency (ns)": 60198243, "97.00 percentile latency (ns)": 71878060, "99.00 percentile latency (ns)": 76373094, "99.90 percentile latency (ns)": 82700434, "Max latency (ns)": 110627829, "Mean latency (ns)": 45906890, "Min duration satisfied": "Yes", "Min latency (ns)": 42209416, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 21.78, "QPS w/o loadgen overhead": 21.78, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.556, "characteristics.good": 36778, "characteristics.power": 7.022400000000008, "characteristics.power.normalized_per_core": 0.058520000000000065, "characteristics.power.normalized_per_processor": 7.022400000000008, "characteristics.samples_per_second": 21.78, "characteristics.samples_per_second.normalized_per_core": 0.18150000000000002, "characteristics.samples_per_second.normalized_per_processor": 21.78, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.4-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.4-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_140.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 21.2766, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "0d3d9a3e5e4e4aa3", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 30656406, "90.00 percentile latency (ns)": 34205190, "90th percentile latency (ns)": 34205190, "95.00 percentile latency (ns)": 44303161, "97.00 percentile latency (ns)": 46621949, "99.00 percentile latency (ns)": 54597868, "99.90 percentile latency (ns)": 58658717, "Max latency (ns)": 65067297, "Mean latency (ns)": 32167271, "Min duration satisfied": "Yes", "Min latency (ns)": 30283265, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 31.07, "QPS w/o loadgen overhead": 31.09, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.764, "characteristics.good": 35382, "characteristics.power": 7.214100000000007, "characteristics.power.normalized_per_core": 0.06011750000000006, "characteristics.power.normalized_per_processor": 7.214100000000007, "characteristics.samples_per_second": 31.09, "characteristics.samples_per_second.normalized_per_core": 0.25908333333333333, "characteristics.samples_per_second.normalized_per_processor": 31.09, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 30.303, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "f4f9523f0e180533", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 7296050, "90.00 percentile latency (ns)": 8689239, "90th percentile latency (ns)": 8689239, "95.00 percentile latency (ns)": 8970309, "97.00 percentile latency (ns)": 9202259, "99.00 percentile latency (ns)": 9791879, "99.90 percentile latency (ns)": 13276972, "Max latency (ns)": 29382676, "Mean latency (ns)": 7618541, "Min duration satisfied": "Yes", "Min latency (ns)": 6820477, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 131.01, "QPS w/o loadgen overhead": 131.26, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 47.81, "characteristics.good": 23905, "characteristics.power": 5.610216666666663, "characteristics.power.normalized_per_core": 0.04675180555555552, "characteristics.power.normalized_per_processor": 5.610216666666663, "characteristics.samples_per_second": 131.26, "characteristics.samples_per_second.normalized_per_core": 1.0938333333333332, "characteristics.samples_per_second.normalized_per_processor": 131.26, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "47b0229e9415d816", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 16977052, "90.00 percentile latency (ns)": 20478638, "90th percentile latency (ns)": 20478638, "95.00 percentile latency (ns)": 22495397, "97.00 percentile latency (ns)": 23098616, "99.00 percentile latency (ns)": 24420640, "99.90 percentile latency (ns)": 28977627, "Max latency (ns)": 36954772, "Mean latency (ns)": 17707605, "Min duration satisfied": "Yes", "Min latency (ns)": 16360272, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 56.42, "QPS w/o loadgen overhead": 56.47, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.038, "characteristics.good": 32019, "characteristics.power": 6.054633333333342, "characteristics.power.normalized_per_core": 0.05045527777777785, "characteristics.power.normalized_per_processor": 6.054633333333342, "characteristics.samples_per_second": 56.47, "characteristics.samples_per_second.normalized_per_core": 0.4705833333333333, "characteristics.samples_per_second.normalized_per_processor": 56.47, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 55.5556, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "e274c40a47179ad6", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 12682246, "90.00 percentile latency (ns)": 14392910, "90th percentile latency (ns)": 14392910, "95.00 percentile latency (ns)": 15802799, "97.00 percentile latency (ns)": 16139445, "99.00 percentile latency (ns)": 16926959, "99.90 percentile latency (ns)": 19844673, "Max latency (ns)": 37163098, "Mean latency (ns)": 13052287, "Min duration satisfied": "Yes", "Min latency (ns)": 12110961, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 76.53, "QPS w/o loadgen overhead": 76.61, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 51.848, "characteristics.good": 25924, "characteristics.power": 5.658116666666676, "characteristics.power.normalized_per_core": 0.0471509722222223, "characteristics.power.normalized_per_processor": 5.658116666666676, "characteristics.samples_per_second": 76.61, "characteristics.samples_per_second.normalized_per_core": 0.6384166666666666, "characteristics.samples_per_second.normalized_per_processor": 76.61, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "54a30e4b49bc2af4", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 9290274, "90.00 percentile latency (ns)": 13364758, "90th percentile latency (ns)": 13364758, "95.00 percentile latency (ns)": 13711323, "97.00 percentile latency (ns)": 14180017, "99.00 percentile latency (ns)": 15048553, "99.90 percentile latency (ns)": 18320091, "Max latency (ns)": 33146019, "Mean latency (ns)": 10225364, "Min duration satisfied": "Yes", "Min latency (ns)": 8588437, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 97.63, "QPS w/o loadgen overhead": 97.8, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.232, "characteristics.good": 29616, "characteristics.power": 6.132299999999993, "characteristics.power.normalized_per_core": 0.05110249999999994, "characteristics.power.normalized_per_processor": 6.132299999999993, "characteristics.samples_per_second": 97.8, "characteristics.samples_per_second.normalized_per_core": 0.815, "characteristics.samples_per_second.normalized_per_processor": 97.8, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "8fb402a898e58763", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 17534110, "90.00 percentile latency (ns)": 21032703, "90th percentile latency (ns)": 21032703, "95.00 percentile latency (ns)": 25948411, "97.00 percentile latency (ns)": 29392215, "99.00 percentile latency (ns)": 30886073, "99.90 percentile latency (ns)": 33981567, "Max latency (ns)": 39664323, "Mean latency (ns)": 18652872, "Min duration satisfied": "Yes", "Min latency (ns)": 16579221, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 53.56, "QPS w/o loadgen overhead": 53.61, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.212, "characteristics.good": 34106, "characteristics.power": 6.888233333333327, "characteristics.power.normalized_per_core": 0.05740194444444439, "characteristics.power.normalized_per_processor": 6.888233333333327, "characteristics.samples_per_second": 53.61, "characteristics.samples_per_second.normalized_per_core": 0.44675, "characteristics.samples_per_second.normalized_per_processor": 53.61, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 52.6316, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "01d8fa3fc4568804", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 8610990, "90.00 percentile latency (ns)": 11372093, "90th percentile latency (ns)": 11372093, "95.00 percentile latency (ns)": 11821080, "97.00 percentile latency (ns)": 12251332, "99.00 percentile latency (ns)": 13339693, "99.90 percentile latency (ns)": 16973966, "Max latency (ns)": 25976123, "Mean latency (ns)": 9255328, "Min duration satisfied": "Yes", "Min latency (ns)": 7965207, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 107.84, "QPS w/o loadgen overhead": 108.05, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.072, "characteristics.good": 29536, "characteristics.power": 6.1663166666666624, "characteristics.power.normalized_per_core": 0.05138597222222219, "characteristics.power.normalized_per_processor": 6.1663166666666624, "characteristics.samples_per_second": 108.05, "characteristics.samples_per_second.normalized_per_core": 0.9004166666666666, "characteristics.samples_per_second.normalized_per_processor": 108.05, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d4527b5307120bcf", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 8254027, "90.00 percentile latency (ns)": 12691292, "90th percentile latency (ns)": 12691292, "95.00 percentile latency (ns)": 12983100, "97.00 percentile latency (ns)": 13371719, "99.00 percentile latency (ns)": 14138079, "99.90 percentile latency (ns)": 17003326, "Max latency (ns)": 32488732, "Mean latency (ns)": 9153424, "Min duration satisfied": "Yes", "Min latency (ns)": 7520685, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 109.06, "QPS w/o loadgen overhead": 109.25, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.386, "characteristics.good": 29193, "characteristics.power": 6.49038333333334, "characteristics.power.normalized_per_core": 0.05408652777777783, "characteristics.power.normalized_per_processor": 6.49038333333334, "characteristics.samples_per_second": 109.25, "characteristics.samples_per_second.normalized_per_core": 0.9104166666666667, "characteristics.samples_per_second.normalized_per_processor": 109.25, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "b428629f8f6f4248", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 29006683, "90.00 percentile latency (ns)": 35020575, "90th percentile latency (ns)": 35020575, "95.00 percentile latency (ns)": 41486092, "97.00 percentile latency (ns)": 45110531, "99.00 percentile latency (ns)": 49006515, "99.90 percentile latency (ns)": 53904049, "Max latency (ns)": 62887955, "Mean latency (ns)": 30676935, "Min duration satisfied": "Yes", "Min latency (ns)": 28307786, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 32.58, "QPS w/o loadgen overhead": 32.6, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.894, "characteristics.good": 36447, "characteristics.power": 6.601349999999991, "characteristics.power.normalized_per_core": 0.05501124999999993, "characteristics.power.normalized_per_processor": 6.601349999999991, "characteristics.samples_per_second": 32.6, "characteristics.samples_per_second.normalized_per_core": 0.27166666666666667, "characteristics.samples_per_second.normalized_per_processor": 32.6, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 31.25, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d9274ac44fdb23f2", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 27934526, "90.00 percentile latency (ns)": 33059172, "90th percentile latency (ns)": 33059172, "95.00 percentile latency (ns)": 40601081, "97.00 percentile latency (ns)": 44430177, "99.00 percentile latency (ns)": 48324479, "99.90 percentile latency (ns)": 53204390, "Max latency (ns)": 59000413, "Mean latency (ns)": 29441204, "Min duration satisfied": "Yes", "Min latency (ns)": 27347022, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 33.95, "QPS w/o loadgen overhead": 33.97, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.598, "characteristics.good": 35299, "characteristics.power": 6.911449999999983, "characteristics.power.normalized_per_core": 0.05759541666666653, "characteristics.power.normalized_per_processor": 6.911449999999983, "characteristics.samples_per_second": 33.97, "characteristics.samples_per_second.normalized_per_core": 0.2830833333333333, "characteristics.samples_per_second.normalized_per_processor": 33.97, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 33.3333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "fc914194b7ba2f12", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 10814881, "90.00 percentile latency (ns)": 15482329, "90th percentile latency (ns)": 15482329, "95.00 percentile latency (ns)": 15992819, "97.00 percentile latency (ns)": 16537525, "99.00 percentile latency (ns)": 17471709, "99.90 percentile latency (ns)": 20885840, "Max latency (ns)": 29647880, "Mean latency (ns)": 11678856, "Min duration satisfied": "Yes", "Min latency (ns)": 10211220, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 85.5, "QPS w/o loadgen overhead": 85.62, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.522, "characteristics.good": 28261, "characteristics.power": 6.270900000000005, "characteristics.power.normalized_per_core": 0.05225750000000004, "characteristics.power.normalized_per_processor": 6.270900000000005, "characteristics.samples_per_second": 85.62, "characteristics.samples_per_second.normalized_per_core": 0.7135, "characteristics.samples_per_second.normalized_per_processor": 85.62, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "395c76d1248c3b64", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 16487462, "90.00 percentile latency (ns)": 23432752, "90th percentile latency (ns)": 23432752, "95.00 percentile latency (ns)": 25982395, "97.00 percentile latency (ns)": 26996327, "99.00 percentile latency (ns)": 28219733, "99.90 percentile latency (ns)": 32237261, "Max latency (ns)": 40756756, "Mean latency (ns)": 18095170, "Min duration satisfied": "Yes", "Min latency (ns)": 15526585, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 55.21, "QPS w/o loadgen overhead": 55.26, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.726, "characteristics.good": 33863, "characteristics.power": 6.512133333333333, "characteristics.power.normalized_per_core": 0.054267777777777776, "characteristics.power.normalized_per_processor": 6.512133333333333, "characteristics.samples_per_second": 55.26, "characteristics.samples_per_second.normalized_per_core": 0.46049999999999996, "characteristics.samples_per_second.normalized_per_processor": 55.26, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 52.6316, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "05a47e5ab4cd8754", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 5060595, "90.00 percentile latency (ns)": 6105979, "90th percentile latency (ns)": 6105979, "95.00 percentile latency (ns)": 6251628, "97.00 percentile latency (ns)": 6407953, "99.00 percentile latency (ns)": 6831296, "99.90 percentile latency (ns)": 10104444, "Max latency (ns)": 27148890, "Mean latency (ns)": 5291032, "Min duration satisfied": "Yes", "Min latency (ns)": 4840011, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 188.51, "QPS w/o loadgen overhead": 189.0, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 44.116, "characteristics.good": 22058, "characteristics.power": 5.556699999999998, "characteristics.power.normalized_per_core": 0.04630583333333332, "characteristics.power.normalized_per_processor": 5.556699999999998, "characteristics.samples_per_second": 189.0, "characteristics.samples_per_second.normalized_per_core": 1.575, "characteristics.samples_per_second.normalized_per_processor": 189.0, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "7e4de2a866932bc4", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 4202050, "90.00 percentile latency (ns)": 5737830, "90th percentile latency (ns)": 5737830, "95.00 percentile latency (ns)": 5914721, "97.00 percentile latency (ns)": 6053439, "99.00 percentile latency (ns)": 6482960, "99.90 percentile latency (ns)": 9505746, "Max latency (ns)": 25911158, "Mean latency (ns)": 4529575, "Min duration satisfied": "Yes", "Min latency (ns)": 3851577, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 220.04, "QPS w/o loadgen overhead": 220.77, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 45.444, "characteristics.good": 22722, "characteristics.power": 5.9742333333333235, "characteristics.power.normalized_per_core": 0.0497852777777777, "characteristics.power.normalized_per_processor": 5.9742333333333235, "characteristics.samples_per_second": 220.77, "characteristics.samples_per_second.normalized_per_core": 1.83975, "characteristics.samples_per_second.normalized_per_processor": 220.77, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d5f84f8745af8b12", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 44123632, "90.00 percentile latency (ns)": 51240277, "90th percentile latency (ns)": 51240277, "95.00 percentile latency (ns)": 57468675, "97.00 percentile latency (ns)": 61738810, "99.00 percentile latency (ns)": 64078364, "99.90 percentile latency (ns)": 69817633, "Max latency (ns)": 76676668, "Mean latency (ns)": 45786754, "Min duration satisfied": "Yes", "Min latency (ns)": 43306040, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 21.83, "QPS w/o loadgen overhead": 21.84, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.624, "characteristics.good": 35312, "characteristics.power": 6.7782000000000115, "characteristics.power.normalized_per_core": 0.0564850000000001, "characteristics.power.normalized_per_processor": 6.7782000000000115, "characteristics.samples_per_second": 21.84, "characteristics.samples_per_second.normalized_per_core": 0.182, "characteristics.samples_per_second.normalized_per_processor": 21.84, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 20.8333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "95e4d46947341e3b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 11266146, "90.00 percentile latency (ns)": 17240799, "90th percentile latency (ns)": 17240799, "95.00 percentile latency (ns)": 17661570, "97.00 percentile latency (ns)": 18308438, "99.00 percentile latency (ns)": 19288067, "99.90 percentile latency (ns)": 22901715, "Max latency (ns)": 37607568, "Mean latency (ns)": 12445264, "Min duration satisfied": "Yes", "Min latency (ns)": 10452200, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 80.25, "QPS w/o loadgen overhead": 80.35, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.442, "characteristics.good": 31721, "characteristics.power": 6.44296666666666, "characteristics.power.normalized_per_core": 0.05369138888888883, "characteristics.power.normalized_per_processor": 6.44296666666666, "characteristics.samples_per_second": 80.35, "characteristics.samples_per_second.normalized_per_core": 0.6695833333333333, "characteristics.samples_per_second.normalized_per_processor": 80.35, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "1a7c401ba336f903", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 22773070, "90.00 percentile latency (ns)": 25002214, "90th percentile latency (ns)": 25002214, "95.00 percentile latency (ns)": 32955866, "97.00 percentile latency (ns)": 37438954, "99.00 percentile latency (ns)": 40955997, "99.90 percentile latency (ns)": 44513624, "Max latency (ns)": 49584983, "Mean latency (ns)": 23861912, "Min duration satisfied": "Yes", "Min latency (ns)": 22425243, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 41.88, "QPS w/o loadgen overhead": 41.91, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.094, "characteristics.good": 35047, "characteristics.power": 7.175666666666651, "characteristics.power.normalized_per_core": 0.05979722222222209, "characteristics.power.normalized_per_processor": 7.175666666666651, "characteristics.samples_per_second": 41.91, "characteristics.samples_per_second.normalized_per_core": 0.34924999999999995, "characteristics.samples_per_second.normalized_per_processor": 41.91, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "399d16ad1e4bc6cb", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 214360323, "90.00 percentile latency (ns)": 230770955, "90th percentile latency (ns)": 230770955, "95.00 percentile latency (ns)": 239611503, "97.00 percentile latency (ns)": 251716889, "99.00 percentile latency (ns)": 329377347, "99.90 percentile latency (ns)": 360095929, "Max latency (ns)": 363027148, "Mean latency (ns)": 221498146, "Min duration satisfied": "Yes", "Min latency (ns)": 212327872, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 4.51, "QPS w/o loadgen overhead": 4.51, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.116, "characteristics.good": 37058, "characteristics.power": 7.0678202995008395, "characteristics.power.normalized_per_core": 0.058898502495840326, "characteristics.power.normalized_per_processor": 7.0678202995008395, "characteristics.samples_per_second": 4.51, "characteristics.samples_per_second.normalized_per_core": 0.03758333333333333, "characteristics.samples_per_second.normalized_per_processor": 4.51, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite2-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 4.52489, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "79ec5dedb00cc2b9", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 7552250, "90.00 percentile latency (ns)": 11090109, "90th percentile latency (ns)": 11090109, "95.00 percentile latency (ns)": 11339733, "97.00 percentile latency (ns)": 11668088, "99.00 percentile latency (ns)": 12377748, "99.90 percentile latency (ns)": 15209115, "Max latency (ns)": 30900240, "Mean latency (ns)": 8271151, "Min duration satisfied": "Yes", "Min latency (ns)": 6875791, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 120.66, "QPS w/o loadgen overhead": 120.9, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.432, "characteristics.good": 29716, "characteristics.power": 6.327450000000008, "characteristics.power.normalized_per_core": 0.05272875000000007, "characteristics.power.normalized_per_processor": 6.327450000000008, "characteristics.samples_per_second": 120.9, "characteristics.samples_per_second.normalized_per_core": 1.0075, "characteristics.samples_per_second.normalized_per_processor": 120.9, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "3425c4ca7b61952b", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 9149567, "90.00 percentile latency (ns)": 9623719, "90th percentile latency (ns)": 9623719, "95.00 percentile latency (ns)": 14543047, "97.00 percentile latency (ns)": 15162695, "99.00 percentile latency (ns)": 15873712, "99.90 percentile latency (ns)": 18742734, "Max latency (ns)": 28152749, "Mean latency (ns)": 9602843, "Min duration satisfied": "Yes", "Min latency (ns)": 8900830, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 103.98, "QPS w/o loadgen overhead": 104.14, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.458, "characteristics.good": 30729, "characteristics.power": 6.677066666666648, "characteristics.power.normalized_per_core": 0.055642222222222065, "characteristics.power.normalized_per_processor": 6.677066666666648, "characteristics.samples_per_second": 104.14, "characteristics.samples_per_second.normalized_per_core": 0.8678333333333333, "characteristics.samples_per_second.normalized_per_processor": 104.14, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6de7bf493fb02b0c", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 13427883, "90.00 percentile latency (ns)": 17408890, "90th percentile latency (ns)": 17408890, "95.00 percentile latency (ns)": 20134221, "97.00 percentile latency (ns)": 20863339, "99.00 percentile latency (ns)": 22056055, "99.90 percentile latency (ns)": 26409014, "Max latency (ns)": 43141567, "Mean latency (ns)": 14293197, "Min duration satisfied": "Yes", "Min latency (ns)": 12830214, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 69.88, "QPS w/o loadgen overhead": 69.96, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.544, "characteristics.good": 29272, "characteristics.power": 6.417866666666661, "characteristics.power.normalized_per_core": 0.053482222222222174, "characteristics.power.normalized_per_processor": 6.417866666666661, "characteristics.samples_per_second": 69.96, "characteristics.samples_per_second.normalized_per_core": 0.583, "characteristics.samples_per_second.normalized_per_processor": 69.96, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "68ec4457e8e30ef1", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 19633524, "90.00 percentile latency (ns)": 23900229, "90th percentile latency (ns)": 23900229, "95.00 percentile latency (ns)": 27573207, "97.00 percentile latency (ns)": 28172247, "99.00 percentile latency (ns)": 29613333, "99.90 percentile latency (ns)": 35124527, "Max latency (ns)": 44563271, "Mean latency (ns)": 20604100, "Min duration satisfied": "Yes", "Min latency (ns)": 18902977, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 48.5, "QPS w/o loadgen overhead": 48.53, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.036, "characteristics.good": 33018, "characteristics.power": 6.151016666666666, "characteristics.power.normalized_per_core": 0.05125847222222222, "characteristics.power.normalized_per_processor": 6.151016666666666, "characteristics.samples_per_second": 48.53, "characteristics.samples_per_second.normalized_per_core": 0.4044166666666667, "characteristics.samples_per_second.normalized_per_processor": 48.53, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 43.4783, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ada0c415f814482d", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 83829787, "90.00 percentile latency (ns)": 98478035, "90th percentile latency (ns)": 98478035, "95.00 percentile latency (ns)": 104373234, "97.00 percentile latency (ns)": 120068451, "99.00 percentile latency (ns)": 124515281, "99.90 percentile latency (ns)": 131719459, "Max latency (ns)": 138211047, "Mean latency (ns)": 87100527, "Min duration satisfied": "Yes", "Min latency (ns)": 82229753, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 11.48, "QPS w/o loadgen overhead": 11.48, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.642, "characteristics.good": 38321, "characteristics.power": 6.583166666666663, "characteristics.power.normalized_per_core": 0.05485972222222219, "characteristics.power.normalized_per_processor": 6.583166666666663, "characteristics.samples_per_second": 11.48, "characteristics.samples_per_second.normalized_per_core": 0.09566666666666666, "characteristics.samples_per_second.normalized_per_processor": 11.48, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite1-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 11.4943, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ed08a01561320270", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 21882481, "90.00 percentile latency (ns)": 26589232, "90th percentile latency (ns)": 26589232, "95.00 percentile latency (ns)": 32136917, "97.00 percentile latency (ns)": 35152772, "99.00 percentile latency (ns)": 36976272, "99.90 percentile latency (ns)": 41522547, "Max latency (ns)": 49967406, "Mean latency (ns)": 23064513, "Min duration satisfied": "Yes", "Min latency (ns)": 20761008, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 43.32, "QPS w/o loadgen overhead": 43.36, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.678, "characteristics.good": 34839, "characteristics.power": 6.738666666666675, "characteristics.power.normalized_per_core": 0.056155555555555625, "characteristics.power.normalized_per_processor": 6.738666666666675, "characteristics.samples_per_second": 43.36, "characteristics.samples_per_second.normalized_per_core": 0.36133333333333334, "characteristics.samples_per_second.normalized_per_processor": 43.36, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 41.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4d050645c8bd5947", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 36232731, "90.00 percentile latency (ns)": 41751166, "90th percentile latency (ns)": 41751166, "95.00 percentile latency (ns)": 50768599, "97.00 percentile latency (ns)": 55629420, "99.00 percentile latency (ns)": 57769004, "99.90 percentile latency (ns)": 61067485, "Max latency (ns)": 65592721, "Mean latency (ns)": 37883957, "Min duration satisfied": "Yes", "Min latency (ns)": 35638967, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 26.39, "QPS w/o loadgen overhead": 26.4, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.262, "characteristics.good": 34631, "characteristics.power": 6.5774333333333255, "characteristics.power.normalized_per_core": 0.05481194444444438, "characteristics.power.normalized_per_processor": 6.5774333333333255, "characteristics.samples_per_second": 26.4, "characteristics.samples_per_second.normalized_per_core": 0.22, "characteristics.samples_per_second.normalized_per_processor": 26.4, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 24.3902, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "a5be6449e9e45a34", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 24558893, "90.00 percentile latency (ns)": 28580378, "90th percentile latency (ns)": 28580378, "95.00 percentile latency (ns)": 35755860, "97.00 percentile latency (ns)": 40090048, "99.00 percentile latency (ns)": 42043720, "99.90 percentile latency (ns)": 47035735, "Max latency (ns)": 53679202, "Mean latency (ns)": 25897128, "Min duration satisfied": "Yes", "Min latency (ns)": 23968903, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 38.59, "QPS w/o loadgen overhead": 38.61, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.76, "characteristics.good": 34380, "characteristics.power": 6.807933333333328, "characteristics.power.normalized_per_core": 0.056732777777777736, "characteristics.power.normalized_per_processor": 6.807933333333328, "characteristics.samples_per_second": 38.61, "characteristics.samples_per_second.normalized_per_core": 0.32175, "characteristics.samples_per_second.normalized_per_processor": 38.61, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 37.037, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d49d095341a5bdce", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 16511179, "90.00 percentile latency (ns)": 18868639, "90th percentile latency (ns)": 18868639, "95.00 percentile latency (ns)": 20886106, "97.00 percentile latency (ns)": 21293540, "99.00 percentile latency (ns)": 22445883, "99.90 percentile latency (ns)": 25850113, "Max latency (ns)": 42356217, "Mean latency (ns)": 17014335, "Min duration satisfied": "Yes", "Min latency (ns)": 15991933, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 58.72, "QPS w/o loadgen overhead": 58.77, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.33, "characteristics.good": 31665, "characteristics.power": 5.7853333333333525, "characteristics.power.normalized_per_core": 0.048211111111111274, "characteristics.power.normalized_per_processor": 5.7853333333333525, "characteristics.samples_per_second": 58.77, "characteristics.samples_per_second.normalized_per_core": 0.48975, "characteristics.samples_per_second.normalized_per_processor": 58.77, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 55.5556, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "1b431f958f268443", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 8727865, "90.00 percentile latency (ns)": 11578539, "90th percentile latency (ns)": 11578539, "95.00 percentile latency (ns)": 12159342, "97.00 percentile latency (ns)": 12565310, "99.00 percentile latency (ns)": 13380840, "99.90 percentile latency (ns)": 16198086, "Max latency (ns)": 31681158, "Mean latency (ns)": 9355861, "Min duration satisfied": "Yes", "Min latency (ns)": 8252994, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 106.72, "QPS w/o loadgen overhead": 106.88, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 57.918, "characteristics.good": 28959, "characteristics.power": 6.013716666666675, "characteristics.power.normalized_per_core": 0.05011430555555563, "characteristics.power.normalized_per_processor": 6.013716666666675, "characteristics.samples_per_second": 106.88, "characteristics.samples_per_second.normalized_per_core": 0.8906666666666666, "characteristics.samples_per_second.normalized_per_processor": 106.88, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d2b9f6b0324d2dab", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 55925288, "90.00 percentile latency (ns)": 65465759, "90th percentile latency (ns)": 65465759, "95.00 percentile latency (ns)": 69840860, "97.00 percentile latency (ns)": 76871660, "99.00 percentile latency (ns)": 79853626, "99.90 percentile latency (ns)": 87847318, "Max latency (ns)": 99614390, "Mean latency (ns)": 57881938, "Min duration satisfied": "Yes", "Min latency (ns)": 54763283, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.27, "QPS w/o loadgen overhead": 17.28, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.674, "characteristics.good": 35837, "characteristics.power": 6.9211480865224395, "characteristics.power.normalized_per_core": 0.05767623405435366, "characteristics.power.normalized_per_processor": 6.9211480865224395, "characteristics.samples_per_second": 17.28, "characteristics.samples_per_second.normalized_per_core": 0.14400000000000002, "characteristics.samples_per_second.normalized_per_processor": 17.28, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 16.129, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "50e1522bc543fea6", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 5369040, "90.00 percentile latency (ns)": 6825258, "90th percentile latency (ns)": 6825258, "95.00 percentile latency (ns)": 7019965, "97.00 percentile latency (ns)": 7257698, "99.00 percentile latency (ns)": 7957715, "99.90 percentile latency (ns)": 11526636, "Max latency (ns)": 26899795, "Mean latency (ns)": 5705417, "Min duration satisfied": "Yes", "Min latency (ns)": 4841349, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 174.75, "QPS w/o loadgen overhead": 175.27, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 43.454, "characteristics.good": 21727, "characteristics.power": 5.821283333333339, "characteristics.power.normalized_per_core": 0.04851069444444449, "characteristics.power.normalized_per_processor": 5.821283333333339, "characteristics.samples_per_second": 175.27, "characteristics.samples_per_second.normalized_per_core": 1.4605833333333333, "characteristics.samples_per_second.normalized_per_processor": 175.27, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "b6949a45c48e1efb", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 264639122, "90.00 percentile latency (ns)": 285170420, "90th percentile latency (ns)": 285170420, "95.00 percentile latency (ns)": 299276097, "97.00 percentile latency (ns)": 353649016, "99.00 percentile latency (ns)": 418448499, "99.90 percentile latency (ns)": 423447616, "Max latency (ns)": 430766958, "Mean latency (ns)": 274857246, "Min duration satisfied": "Yes", "Min latency (ns)": 261124786, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 3.64, "QPS w/o loadgen overhead": 3.64, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 80.676, "characteristics.good": 40338, "characteristics.power": 6.889633333333328, "characteristics.power.normalized_per_core": 0.05741361111111107, "characteristics.power.normalized_per_processor": 6.889633333333328, "characteristics.samples_per_second": 3.64, "characteristics.samples_per_second.normalized_per_core": 0.030333333333333334, "characteristics.samples_per_second.normalized_per_processor": 3.64, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite4-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.62319, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "99a77e12c4a4a9c6", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 22599671, "90.00 percentile latency (ns)": 26264949, "90th percentile latency (ns)": 26264949, "95.00 percentile latency (ns)": 32771581, "97.00 percentile latency (ns)": 36818438, "99.00 percentile latency (ns)": 38952423, "99.90 percentile latency (ns)": 42777091, "Max latency (ns)": 50834677, "Mean latency (ns)": 23774223, "Min duration satisfied": "Yes", "Min latency (ns)": 21899763, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 42.03, "QPS w/o loadgen overhead": 42.06, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.33, "characteristics.good": 35165, "characteristics.power": 6.751900000000014, "characteristics.power.normalized_per_core": 0.05626583333333345, "characteristics.power.normalized_per_processor": 6.751900000000014, "characteristics.samples_per_second": 42.06, "characteristics.samples_per_second.normalized_per_core": 0.35050000000000003, "characteristics.samples_per_second.normalized_per_processor": 42.06, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 41.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "827a6c11469b8ebe", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 18361657, "90.00 percentile latency (ns)": 21187816, "90th percentile latency (ns)": 21187816, "95.00 percentile latency (ns)": 26901300, "97.00 percentile latency (ns)": 30351131, "99.00 percentile latency (ns)": 31701952, "99.90 percentile latency (ns)": 35561697, "Max latency (ns)": 40154540, "Mean latency (ns)": 19304104, "Min duration satisfied": "Yes", "Min latency (ns)": 17988299, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 51.76, "QPS w/o loadgen overhead": 51.8, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.4, "characteristics.good": 33700, "characteristics.power": 6.759083333333348, "characteristics.power.normalized_per_core": 0.05632569444444457, "characteristics.power.normalized_per_processor": 6.759083333333348, "characteristics.samples_per_second": 51.8, "characteristics.samples_per_second.normalized_per_core": 0.43166666666666664, "characteristics.samples_per_second.normalized_per_processor": 51.8, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 47.619, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "900f50d4144983b2", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 15283781, "90.00 percentile latency (ns)": 16466029, "90th percentile latency (ns)": 16466029, "95.00 percentile latency (ns)": 22304717, "97.00 percentile latency (ns)": 25919025, "99.00 percentile latency (ns)": 27117736, "99.90 percentile latency (ns)": 30374499, "Max latency (ns)": 40244704, "Mean latency (ns)": 16014256, "Min duration satisfied": "Yes", "Min latency (ns)": 14961597, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 62.39, "QPS w/o loadgen overhead": 62.44, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.208, "characteristics.good": 33604, "characteristics.power": 6.974066666666686, "characteristics.power.normalized_per_core": 0.05811722222222238, "characteristics.power.normalized_per_processor": 6.974066666666686, "characteristics.samples_per_second": 62.44, "characteristics.samples_per_second.normalized_per_core": 0.5203333333333333, "characteristics.samples_per_second.normalized_per_processor": 62.44, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 55.5556, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "f3be79fb19ed9464", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 22983640, "90.00 percentile latency (ns)": 26622024, "90th percentile latency (ns)": 26622024, "95.00 percentile latency (ns)": 31949181, "97.00 percentile latency (ns)": 32754751, "99.00 percentile latency (ns)": 33888317, "99.90 percentile latency (ns)": 37455064, "Max latency (ns)": 49860803, "Mean latency (ns)": 23875620, "Min duration satisfied": "Yes", "Min latency (ns)": 22378665, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 41.86, "QPS w/o loadgen overhead": 41.88, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.348, "characteristics.good": 32174, "characteristics.power": 6.161083333333326, "characteristics.power.normalized_per_core": 0.05134236111111105, "characteristics.power.normalized_per_processor": 6.161083333333326, "characteristics.samples_per_second": 41.88, "characteristics.samples_per_second.normalized_per_core": 0.34900000000000003, "characteristics.samples_per_second.normalized_per_processor": 41.88, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "744faa20344fe9d8", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 17367710, "90.00 percentile latency (ns)": 18961117, "90th percentile latency (ns)": 18961117, "95.00 percentile latency (ns)": 24372676, "97.00 percentile latency (ns)": 25027754, "99.00 percentile latency (ns)": 26146986, "99.90 percentile latency (ns)": 29925400, "Max latency (ns)": 44977182, "Mean latency (ns)": 18005547, "Min duration satisfied": "Yes", "Min latency (ns)": 16853184, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 55.5, "QPS w/o loadgen overhead": 55.54, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.246, "characteristics.good": 31623, "characteristics.power": 6.120166666666674, "characteristics.power.normalized_per_core": 0.05100138888888895, "characteristics.power.normalized_per_processor": 6.120166666666674, "characteristics.samples_per_second": 55.54, "characteristics.samples_per_second.normalized_per_core": 0.4628333333333333, "characteristics.samples_per_second.normalized_per_processor": 55.54, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c9f925b3bfce45b9", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 14543176, "90.00 percentile latency (ns)": 17730493, "90th percentile latency (ns)": 17730493, "95.00 percentile latency (ns)": 19054482, "97.00 percentile latency (ns)": 19480936, "99.00 percentile latency (ns)": 20745370, "99.90 percentile latency (ns)": 24723244, "Max latency (ns)": 36385503, "Mean latency (ns)": 15200031, "Min duration satisfied": "Yes", "Min latency (ns)": 13941727, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 65.72, "QPS w/o loadgen overhead": 65.79, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.668, "characteristics.good": 30834, "characteristics.power": 5.866883333333335, "characteristics.power.normalized_per_core": 0.04889069444444446, "characteristics.power.normalized_per_processor": 5.866883333333335, "characteristics.samples_per_second": 65.79, "characteristics.samples_per_second.normalized_per_core": 0.54825, "characteristics.samples_per_second.normalized_per_processor": 65.79, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 62.5, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c2704707e4a64402", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 20107007, "90.00 percentile latency (ns)": 24314556, "90th percentile latency (ns)": 24314556, "95.00 percentile latency (ns)": 29686093, "97.00 percentile latency (ns)": 31114526, "99.00 percentile latency (ns)": 32354678, "99.90 percentile latency (ns)": 36128530, "Max latency (ns)": 41895458, "Mean latency (ns)": 21105938, "Min duration satisfied": "Yes", "Min latency (ns)": 19667978, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 47.35, "QPS w/o loadgen overhead": 47.38, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.384, "characteristics.good": 33192, "characteristics.power": 6.442099999999983, "characteristics.power.normalized_per_core": 0.053684166666666526, "characteristics.power.normalized_per_processor": 6.442099999999983, "characteristics.samples_per_second": 47.38, "characteristics.samples_per_second.normalized_per_core": 0.39483333333333337, "characteristics.samples_per_second.normalized_per_processor": 47.38, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 43.4783, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6074c8287e865d55", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 5657595, "90.00 percentile latency (ns)": 7188394, "90th percentile latency (ns)": 7188394, "95.00 percentile latency (ns)": 7424797, "97.00 percentile latency (ns)": 7687699, "99.00 percentile latency (ns)": 8426904, "99.90 percentile latency (ns)": 12078693, "Max latency (ns)": 23540939, "Mean latency (ns)": 6018373, "Min duration satisfied": "Yes", "Min latency (ns)": 5042016, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 165.7, "QPS w/o loadgen overhead": 166.16, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.296, "characteristics.good": 24648, "characteristics.power": 5.838299999999992, "characteristics.power.normalized_per_core": 0.04865249999999994, "characteristics.power.normalized_per_processor": 5.838299999999992, "characteristics.samples_per_second": 166.16, "characteristics.samples_per_second.normalized_per_core": 1.3846666666666667, "characteristics.samples_per_second.normalized_per_processor": 166.16, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "41e5b2ec1b0a942d", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 10238449, "90.00 percentile latency (ns)": 15465253, "90th percentile latency (ns)": 15465253, "95.00 percentile latency (ns)": 15753875, "97.00 percentile latency (ns)": 16277843, "99.00 percentile latency (ns)": 17251840, "99.90 percentile latency (ns)": 20705131, "Max latency (ns)": 36434064, "Mean latency (ns)": 11324144, "Min duration satisfied": "Yes", "Min latency (ns)": 9511261, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 88.18, "QPS w/o loadgen overhead": 88.31, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 62.082, "characteristics.good": 31041, "characteristics.power": 6.319833333333334, "characteristics.power.normalized_per_core": 0.05266527777777778, "characteristics.power.normalized_per_processor": 6.319833333333334, "characteristics.samples_per_second": 88.31, "characteristics.samples_per_second.normalized_per_core": 0.7359166666666667, "characteristics.samples_per_second.normalized_per_processor": 88.31, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4fadd794d1eb4f37", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 12530180, "90.00 percentile latency (ns)": 16045167, "90th percentile latency (ns)": 16045167, "95.00 percentile latency (ns)": 17424760, "97.00 percentile latency (ns)": 17888378, "99.00 percentile latency (ns)": 19223439, "99.90 percentile latency (ns)": 23104760, "Max latency (ns)": 29552601, "Mean latency (ns)": 13278291, "Min duration satisfied": "Yes", "Min latency (ns)": 11914775, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 75.23, "QPS w/o loadgen overhead": 75.31, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.224, "characteristics.good": 30612, "characteristics.power": 6.071266666666668, "characteristics.power.normalized_per_core": 0.050593888888888904, "characteristics.power.normalized_per_processor": 6.071266666666668, "characteristics.samples_per_second": 75.31, "characteristics.samples_per_second.normalized_per_core": 0.6275833333333334, "characteristics.samples_per_second.normalized_per_processor": 75.31, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "067811dfe39fd53b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 2984191, "90.00 percentile latency (ns)": 3804321, "90th percentile latency (ns)": 3804321, "95.00 percentile latency (ns)": 4132850, "97.00 percentile latency (ns)": 4212330, "99.00 percentile latency (ns)": 4454343, "99.90 percentile latency (ns)": 7253890, "Max latency (ns)": 23190657, "Mean latency (ns)": 3120435, "Min duration satisfied": "Yes", "Min latency (ns)": 2665531, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 319.12, "QPS w/o loadgen overhead": 320.47, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 42.348, "characteristics.good": 21174, "characteristics.power": 5.9130999999999885, "characteristics.power.normalized_per_core": 0.049275833333333234, "characteristics.power.normalized_per_processor": 5.9130999999999885, "characteristics.samples_per_second": 320.47, "characteristics.samples_per_second.normalized_per_core": 2.6705833333333335, "characteristics.samples_per_second.normalized_per_processor": 320.47, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "da84bb4acad8b700", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 71398143, "90.00 percentile latency (ns)": 85553942, "90th percentile latency (ns)": 85553942, "95.00 percentile latency (ns)": 91482459, "97.00 percentile latency (ns)": 102440989, "99.00 percentile latency (ns)": 106189614, "99.90 percentile latency (ns)": 110471665, "Max latency (ns)": 114170974, "Mean latency (ns)": 74592103, "Min duration satisfied": "Yes", "Min latency (ns)": 70595898, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 13.4, "QPS w/o loadgen overhead": 13.41, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.012, "characteristics.good": 37506, "characteristics.power": 6.41771666666666, "characteristics.power.normalized_per_core": 0.053480972222222166, "characteristics.power.normalized_per_processor": 6.41771666666666, "characteristics.samples_per_second": 13.41, "characteristics.samples_per_second.normalized_per_core": 0.11175, "characteristics.samples_per_second.normalized_per_processor": 13.41, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.3-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.3-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.3_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 13.3333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "a050d384702bd428", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 12677029, "90.00 percentile latency (ns)": 17343769, "90th percentile latency (ns)": 17343769, "95.00 percentile latency (ns)": 18915439, "97.00 percentile latency (ns)": 19694636, "99.00 percentile latency (ns)": 20830296, "99.90 percentile latency (ns)": 25139350, "Max latency (ns)": 40680253, "Mean latency (ns)": 13609334, "Min duration satisfied": "Yes", "Min latency (ns)": 12028638, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 73.39, "QPS w/o loadgen overhead": 73.48, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.062, "characteristics.good": 32031, "characteristics.power": 6.224400000000006, "characteristics.power.normalized_per_core": 0.051870000000000055, "characteristics.power.normalized_per_processor": 6.224400000000006, "characteristics.samples_per_second": 73.48, "characteristics.samples_per_second.normalized_per_core": 0.6123333333333334, "characteristics.samples_per_second.normalized_per_processor": 73.48, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "9a809453ab7b08eb", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 107023392, "90.00 percentile latency (ns)": 122852206, "90th percentile latency (ns)": 122852206, "95.00 percentile latency (ns)": 127200496, "97.00 percentile latency (ns)": 151239267, "99.00 percentile latency (ns)": 164854322, "99.90 percentile latency (ns)": 172605437, "Max latency (ns)": 176170620, "Mean latency (ns)": 111604824, "Min duration satisfied": "Yes", "Min latency (ns)": 106025248, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 8.96, "QPS w/o loadgen overhead": 8.96, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 77.542, "characteristics.good": 38771, "characteristics.power": 6.697933333333314, "characteristics.power.normalized_per_core": 0.05581611111111095, "characteristics.power.normalized_per_processor": 6.697933333333314, "characteristics.samples_per_second": 8.96, "characteristics.samples_per_second.normalized_per_core": 0.07466666666666667, "characteristics.samples_per_second.normalized_per_processor": 8.96, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite2-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 8.69565, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "35b9911ba24b3eab", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 5369935, "90.00 percentile latency (ns)": 7707324, "90th percentile latency (ns)": 7707324, "95.00 percentile latency (ns)": 7966370, "97.00 percentile latency (ns)": 8177990, "99.00 percentile latency (ns)": 8777109, "99.90 percentile latency (ns)": 11794662, "Max latency (ns)": 29537837, "Mean latency (ns)": 5829649, "Min duration satisfied": "Yes", "Min latency (ns)": 4867150, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 171.08, "QPS w/o loadgen overhead": 171.54, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.738, "characteristics.good": 28369, "characteristics.power": 6.244166666666669, "characteristics.power.normalized_per_core": 0.052034722222222246, "characteristics.power.normalized_per_processor": 6.244166666666669, "characteristics.samples_per_second": 171.54, "characteristics.samples_per_second.normalized_per_core": 1.4295, "characteristics.samples_per_second.normalized_per_processor": 171.54, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "7b54e74908be4c41", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 7762233, "90.00 percentile latency (ns)": 10320504, "90th percentile latency (ns)": 10320504, "95.00 percentile latency (ns)": 10637323, "97.00 percentile latency (ns)": 10930502, "99.00 percentile latency (ns)": 11972350, "99.90 percentile latency (ns)": 15465926, "Max latency (ns)": 29719019, "Mean latency (ns)": 8331354, "Min duration satisfied": "Yes", "Min latency (ns)": 7079679, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 119.8, "QPS w/o loadgen overhead": 120.03, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 57.054, "characteristics.good": 28527, "characteristics.power": 6.0990000000000055, "characteristics.power.normalized_per_core": 0.050825000000000044, "characteristics.power.normalized_per_processor": 6.0990000000000055, "characteristics.samples_per_second": 120.03, "characteristics.samples_per_second.normalized_per_core": 1.00025, "characteristics.samples_per_second.normalized_per_processor": 120.03, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "9aa4ad688e09d1c9", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 75343974, "90.00 percentile latency (ns)": 88851667, "90th percentile latency (ns)": 88851667, "95.00 percentile latency (ns)": 96654508, "97.00 percentile latency (ns)": 109259055, "99.00 percentile latency (ns)": 113082266, "99.90 percentile latency (ns)": 117816546, "Max latency (ns)": 120882422, "Mean latency (ns)": 78640556, "Min duration satisfied": "Yes", "Min latency (ns)": 74476808, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 12.71, "QPS w/o loadgen overhead": 12.72, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.69, "characteristics.good": 37845, "characteristics.power": 6.508183333333347, "characteristics.power.normalized_per_core": 0.05423486111111123, "characteristics.power.normalized_per_processor": 6.508183333333347, "characteristics.samples_per_second": 12.72, "characteristics.samples_per_second.normalized_per_core": 0.10600000000000001, "characteristics.samples_per_second.normalized_per_processor": 12.72, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.4-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.4-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.4_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.3457, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "044c765c5e4f20ba", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 7607885, "90.00 percentile latency (ns)": 9232329, "90th percentile latency (ns)": 9232329, "95.00 percentile latency (ns)": 9804150, "97.00 percentile latency (ns)": 10089236, "99.00 percentile latency (ns)": 10940385, "99.90 percentile latency (ns)": 13966742, "Max latency (ns)": 27412347, "Mean latency (ns)": 7925615, "Min duration satisfied": "Yes", "Min latency (ns)": 7003935, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 125.94, "QPS w/o loadgen overhead": 126.17, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 52.446, "characteristics.good": 26223, "characteristics.power": 5.7981833333333395, "characteristics.power.normalized_per_core": 0.0483181944444445, "characteristics.power.normalized_per_processor": 5.7981833333333395, "characteristics.samples_per_second": 126.17, "characteristics.samples_per_second.normalized_per_core": 1.0514166666666667, "characteristics.samples_per_second.normalized_per_processor": 126.17, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "20eeb34cc7933fc0", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 43786582, "90.00 percentile latency (ns)": 53141720, "90th percentile latency (ns)": 53141720, "95.00 percentile latency (ns)": 58787936, "97.00 percentile latency (ns)": 61940607, "99.00 percentile latency (ns)": 64508140, "99.90 percentile latency (ns)": 71159294, "Max latency (ns)": 79710807, "Mean latency (ns)": 45775020, "Min duration satisfied": "Yes", "Min latency (ns)": 43156571, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 21.84, "QPS w/o loadgen overhead": 21.85, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.724, "characteristics.good": 36862, "characteristics.power": 6.2582166666666765, "characteristics.power.normalized_per_core": 0.05215180555555564, "characteristics.power.normalized_per_processor": 6.2582166666666765, "characteristics.samples_per_second": 21.85, "characteristics.samples_per_second.normalized_per_core": 0.18208333333333335, "characteristics.samples_per_second.normalized_per_processor": 21.85, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 21.2766, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "de31b6abf994e76a", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 20784521, "90.00 percentile latency (ns)": 25086816, "90th percentile latency (ns)": 25086816, "95.00 percentile latency (ns)": 30033670, "97.00 percentile latency (ns)": 31005827, "99.00 percentile latency (ns)": 32445656, "99.90 percentile latency (ns)": 37521657, "Max latency (ns)": 45150692, "Mean latency (ns)": 21850465, "Min duration satisfied": "Yes", "Min latency (ns)": 20196216, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 45.73, "QPS w/o loadgen overhead": 45.77, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.604, "characteristics.good": 33302, "characteristics.power": 6.714449999999997, "characteristics.power.normalized_per_core": 0.055953749999999976, "characteristics.power.normalized_per_processor": 6.714449999999997, "characteristics.samples_per_second": 45.77, "characteristics.samples_per_second.normalized_per_core": 0.3814166666666667, "characteristics.samples_per_second.normalized_per_processor": 45.77, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 43.4783, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d8a91a89494ad39a", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 31359651, "90.00 percentile latency (ns)": 35714673, "90th percentile latency (ns)": 35714673, "95.00 percentile latency (ns)": 44711561, "97.00 percentile latency (ns)": 47817044, "99.00 percentile latency (ns)": 51563083, "99.90 percentile latency (ns)": 54883682, "Max latency (ns)": 58494117, "Mean latency (ns)": 32809212, "Min duration satisfied": "Yes", "Min latency (ns)": 30721207, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 30.47, "QPS w/o loadgen overhead": 30.48, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.106, "characteristics.good": 34553, "characteristics.power": 6.601733333333334, "characteristics.power.normalized_per_core": 0.05501444444444445, "characteristics.power.normalized_per_processor": 6.601733333333334, "characteristics.samples_per_second": 30.48, "characteristics.samples_per_second.normalized_per_core": 0.254, "characteristics.samples_per_second.normalized_per_processor": 30.48, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 28.5714, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "624174b2edefcfff", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 10908725, "90.00 percentile latency (ns)": 13427874, "90th percentile latency (ns)": 13427874, "95.00 percentile latency (ns)": 14653162, "97.00 percentile latency (ns)": 15053894, "99.00 percentile latency (ns)": 16255942, "99.90 percentile latency (ns)": 19742995, "Max latency (ns)": 28704726, "Mean latency (ns)": 11446815, "Min duration satisfied": "Yes", "Min latency (ns)": 10288665, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 87.24, "QPS w/o loadgen overhead": 87.36, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.822, "characteristics.good": 29911, "characteristics.power": 6.004783333333349, "characteristics.power.normalized_per_core": 0.05003986111111124, "characteristics.power.normalized_per_processor": 6.004783333333349, "characteristics.samples_per_second": 87.36, "characteristics.samples_per_second.normalized_per_core": 0.728, "characteristics.samples_per_second.normalized_per_processor": 87.36, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "047ddae3a72cd099", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 27490965, "90.00 percentile latency (ns)": 32967103, "90th percentile latency (ns)": 32967103, "95.00 percentile latency (ns)": 37333518, "97.00 percentile latency (ns)": 38670856, "99.00 percentile latency (ns)": 40191216, "99.90 percentile latency (ns)": 45658881, "Max latency (ns)": 53060696, "Mean latency (ns)": 28683791, "Min duration satisfied": "Yes", "Min latency (ns)": 26903682, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 34.84, "QPS w/o loadgen overhead": 34.86, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.336, "characteristics.good": 34168, "characteristics.power": 6.128352745424297, "characteristics.power.normalized_per_core": 0.051069606211869144, "characteristics.power.normalized_per_processor": 6.128352745424297, "characteristics.samples_per_second": 34.86, "characteristics.samples_per_second.normalized_per_core": 0.2905, "characteristics.samples_per_second.normalized_per_processor": 34.86, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 34.4828, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4a778b78b6877d44", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 32811822, "90.00 percentile latency (ns)": 39354455, "90th percentile latency (ns)": 39354455, "95.00 percentile latency (ns)": 43283370, "97.00 percentile latency (ns)": 44136602, "99.00 percentile latency (ns)": 46464025, "99.90 percentile latency (ns)": 52271006, "Max latency (ns)": 62635165, "Mean latency (ns)": 34077042, "Min duration satisfied": "Yes", "Min latency (ns)": 32008302, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 29.33, "QPS w/o loadgen overhead": 29.35, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.834, "characteristics.good": 34417, "characteristics.power": 6.139483333333342, "characteristics.power.normalized_per_core": 0.05116236111111118, "characteristics.power.normalized_per_processor": 6.139483333333342, "characteristics.samples_per_second": 29.35, "characteristics.samples_per_second.normalized_per_core": 0.24458333333333335, "characteristics.samples_per_second.normalized_per_processor": 29.35, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 25, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "269dd275c74b369b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 98325927, "90.00 percentile latency (ns)": 113342719, "90th percentile latency (ns)": 113342719, "95.00 percentile latency (ns)": 116231432, "97.00 percentile latency (ns)": 121350150, "99.00 percentile latency (ns)": 173616720, "99.90 percentile latency (ns)": 182869552, "Max latency (ns)": 183990820, "Mean latency (ns)": 102507920, "Min duration satisfied": "Yes", "Min latency (ns)": 97286704, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 9.75, "QPS w/o loadgen overhead": 9.76, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.254, "characteristics.good": 35627, "characteristics.power": 7.100566666666666, "characteristics.power.normalized_per_core": 0.05917138888888888, "characteristics.power.normalized_per_processor": 7.100566666666666, "characteristics.samples_per_second": 9.76, "characteristics.samples_per_second.normalized_per_core": 0.08133333333333333, "characteristics.samples_per_second.normalized_per_processor": 9.76, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite0-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 9.80392, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "3e7fd2d4e6714b4f", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 157517677, "90.00 percentile latency (ns)": 175505269, "90th percentile latency (ns)": 175505269, "95.00 percentile latency (ns)": 186579154, "97.00 percentile latency (ns)": 218420710, "99.00 percentile latency (ns)": 250141194, "99.90 percentile latency (ns)": 258385579, "Max latency (ns)": 260213769, "Mean latency (ns)": 164399218, "Min duration satisfied": "Yes", "Min latency (ns)": 155996112, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 6.08, "QPS w/o loadgen overhead": 6.08, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 79.268, "characteristics.good": 39634, "characteristics.power": 6.756766666666678, "characteristics.power.normalized_per_core": 0.05630638888888898, "characteristics.power.normalized_per_processor": 6.756766666666678, "characteristics.samples_per_second": 6.08, "characteristics.samples_per_second.normalized_per_core": 0.050666666666666665, "characteristics.samples_per_second.normalized_per_processor": 6.08, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite3-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 6.21118, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ba7253ca048b112d", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 24168786, "90.00 percentile latency (ns)": 28946381, "90th percentile latency (ns)": 28946381, "95.00 percentile latency (ns)": 32793092, "97.00 percentile latency (ns)": 33731852, "99.00 percentile latency (ns)": 35297535, "99.90 percentile latency (ns)": 39910475, "Max latency (ns)": 50230787, "Mean latency (ns)": 25197008, "Min duration satisfied": "Yes", "Min latency (ns)": 23608701, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 39.66, "QPS w/o loadgen overhead": 39.69, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.216, "characteristics.good": 33108, "characteristics.power": 6.065316666666671, "characteristics.power.normalized_per_core": 0.05054430555555559, "characteristics.power.normalized_per_processor": 6.065316666666671, "characteristics.samples_per_second": 39.69, "characteristics.samples_per_second.normalized_per_core": 0.33075, "characteristics.samples_per_second.normalized_per_processor": 39.69, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 38.4615, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "815a74ef441db363", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 302369450, "90.00 percentile latency (ns)": 321198499, "90th percentile latency (ns)": 321198499, "95.00 percentile latency (ns)": 330538153, "97.00 percentile latency (ns)": 377867625, "99.00 percentile latency (ns)": 444627621, "99.90 percentile latency (ns)": 461026500, "Max latency (ns)": 463345517, "Mean latency (ns)": 310545679, "Min duration satisfied": "Yes", "Min latency (ns)": 297509204, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 3.22, "QPS w/o loadgen overhead": 3.22, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.348, "characteristics.good": 38174, "characteristics.power": 7.143183333333324, "characteristics.power.normalized_per_core": 0.0595265277777777, "characteristics.power.normalized_per_processor": 7.143183333333324, "characteristics.samples_per_second": 3.22, "characteristics.samples_per_second.normalized_per_core": 0.026833333333333334, "characteristics.samples_per_second.normalized_per_processor": 3.22, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite3-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.24675, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "14541a0034e4af4d", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 8785596, "90.00 percentile latency (ns)": 12261707, "90th percentile latency (ns)": 12261707, "95.00 percentile latency (ns)": 12598011, "97.00 percentile latency (ns)": 12996511, "99.00 percentile latency (ns)": 13820306, "99.90 percentile latency (ns)": 16834724, "Max latency (ns)": 30222830, "Mean latency (ns)": 9649556, "Min duration satisfied": "Yes", "Min latency (ns)": 8052927, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 103.44, "QPS w/o loadgen overhead": 103.63, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 54.504, "characteristics.good": 27252, "characteristics.power": 6.076316666666667, "characteristics.power.normalized_per_core": 0.05063597222222223, "characteristics.power.normalized_per_processor": 6.076316666666667, "characteristics.samples_per_second": 103.63, "characteristics.samples_per_second.normalized_per_core": 0.8635833333333333, "characteristics.samples_per_second.normalized_per_processor": 103.63, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "56345844e4cadb2a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 9527917, "90.00 percentile latency (ns)": 11316230, "90th percentile latency (ns)": 11316230, "95.00 percentile latency (ns)": 11839083, "97.00 percentile latency (ns)": 12139822, "99.00 percentile latency (ns)": 12878643, "99.90 percentile latency (ns)": 16551283, "Max latency (ns)": 33013202, "Mean latency (ns)": 9819079, "Min duration satisfied": "Yes", "Min latency (ns)": 9051517, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 101.69, "QPS w/o loadgen overhead": 101.84, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.478, "characteristics.good": 24739, "characteristics.power": 5.644733333333343, "characteristics.power.normalized_per_core": 0.04703944444444452, "characteristics.power.normalized_per_processor": 5.644733333333343, "characteristics.samples_per_second": 101.84, "characteristics.samples_per_second.normalized_per_core": 0.8486666666666667, "characteristics.samples_per_second.normalized_per_processor": 101.84, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c34c9d9342accfc2", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 20993560, "90.00 percentile latency (ns)": 23941310, "90th percentile latency (ns)": 23941310, "95.00 percentile latency (ns)": 30718384, "97.00 percentile latency (ns)": 34887811, "99.00 percentile latency (ns)": 36295818, "99.90 percentile latency (ns)": 39354177, "Max latency (ns)": 47658884, "Mean latency (ns)": 22133365, "Min duration satisfied": "Yes", "Min latency (ns)": 20428072, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 45.15, "QPS w/o loadgen overhead": 45.18, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.124, "characteristics.good": 34062, "characteristics.power": 6.965950000000006, "characteristics.power.normalized_per_core": 0.058049583333333384, "characteristics.power.normalized_per_processor": 6.965950000000006, "characteristics.samples_per_second": 45.18, "characteristics.samples_per_second.normalized_per_core": 0.3765, "characteristics.samples_per_second.normalized_per_processor": 45.18, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 45.4545, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "0e47749078a9f1d9", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 55074158, "90.00 percentile latency (ns)": 67372790, "90th percentile latency (ns)": 67372790, "95.00 percentile latency (ns)": 72258975, "97.00 percentile latency (ns)": 79808903, "99.00 percentile latency (ns)": 83472901, "99.90 percentile latency (ns)": 90295456, "Max latency (ns)": 99474654, "Mean latency (ns)": 57733684, "Min duration satisfied": "Yes", "Min latency (ns)": 54385655, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.32, "QPS w/o loadgen overhead": 17.32, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.76, "characteristics.good": 37880, "characteristics.power": 6.2943261231281245, "characteristics.power.normalized_per_core": 0.05245271769273437, "characteristics.power.normalized_per_processor": 6.2943261231281245, "characteristics.samples_per_second": 17.32, "characteristics.samples_per_second.normalized_per_core": 0.14433333333333334, "characteristics.samples_per_second.normalized_per_processor": 17.32, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.1818, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6ed677584703abd8", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 26182363, "90.00 percentile latency (ns)": 30914564, "90th percentile latency (ns)": 30914564, "95.00 percentile latency (ns)": 34109001, "97.00 percentile latency (ns)": 34806042, "99.00 percentile latency (ns)": 36416306, "99.90 percentile latency (ns)": 41665182, "Max latency (ns)": 54014878, "Mean latency (ns)": 27111926, "Min duration satisfied": "Yes", "Min latency (ns)": 25466804, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 36.86, "QPS w/o loadgen overhead": 36.88, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 65.75, "characteristics.good": 32875, "characteristics.power": 5.919149999999989, "characteristics.power.normalized_per_core": 0.049326249999999905, "characteristics.power.normalized_per_processor": 5.919149999999989, "characteristics.samples_per_second": 36.88, "characteristics.samples_per_second.normalized_per_core": 0.30733333333333335, "characteristics.samples_per_second.normalized_per_processor": 36.88, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 33.3333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "0b701a6248028cbf", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 11648504, "90.00 percentile latency (ns)": 17798392, "90th percentile latency (ns)": 17798392, "95.00 percentile latency (ns)": 19181961, "97.00 percentile latency (ns)": 19717610, "99.00 percentile latency (ns)": 20562949, "99.90 percentile latency (ns)": 23652176, "Max latency (ns)": 29965598, "Mean latency (ns)": 12818778, "Min duration satisfied": "Yes", "Min latency (ns)": 10673814, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 77.9, "QPS w/o loadgen overhead": 78.01, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.766, "characteristics.good": 32383, "characteristics.power": 6.775466666666671, "characteristics.power.normalized_per_core": 0.05646222222222226, "characteristics.power.normalized_per_processor": 6.775466666666671, "characteristics.samples_per_second": 78.01, "characteristics.samples_per_second.normalized_per_core": 0.6500833333333333, "characteristics.samples_per_second.normalized_per_processor": 78.01, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4a74a96afffa4142", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 43765641, "90.00 percentile latency (ns)": 52537930, "90th percentile latency (ns)": 52537930, "95.00 percentile latency (ns)": 56850120, "97.00 percentile latency (ns)": 57728261, "99.00 percentile latency (ns)": 60022011, "99.90 percentile latency (ns)": 66844117, "Max latency (ns)": 69698708, "Mean latency (ns)": 45389772, "Min duration satisfied": "Yes", "Min latency (ns)": 42307654, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 22.02, "QPS w/o loadgen overhead": 22.03, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.122, "characteristics.good": 35061, "characteristics.power": 6.177716666666657, "characteristics.power.normalized_per_core": 0.05148097222222214, "characteristics.power.normalized_per_processor": 6.177716666666657, "characteristics.samples_per_second": 22.03, "characteristics.samples_per_second.normalized_per_core": 0.18358333333333335, "characteristics.samples_per_second.normalized_per_processor": 22.03, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 19.2308, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "dc4c454889b93c09", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 11479255, "90.00 percentile latency (ns)": 13761360, "90th percentile latency (ns)": 13761360, "95.00 percentile latency (ns)": 18559000, "97.00 percentile latency (ns)": 19294262, "99.00 percentile latency (ns)": 20159800, "99.90 percentile latency (ns)": 23160534, "Max latency (ns)": 30268167, "Mean latency (ns)": 12133521, "Min duration satisfied": "Yes", "Min latency (ns)": 11110824, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 82.31, "QPS w/o loadgen overhead": 82.42, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.128, "characteristics.good": 32064, "characteristics.power": 6.805016666666672, "characteristics.power.normalized_per_core": 0.05670847222222227, "characteristics.power.normalized_per_processor": 6.805016666666672, "characteristics.samples_per_second": 82.42, "characteristics.samples_per_second.normalized_per_core": 0.6868333333333333, "characteristics.samples_per_second.normalized_per_processor": 82.42, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "dc9b2f7bc6d4e90a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 19439990, "90.00 percentile latency (ns)": 23407475, "90th percentile latency (ns)": 23407475, "95.00 percentile latency (ns)": 25280227, "97.00 percentile latency (ns)": 25820661, "99.00 percentile latency (ns)": 27286961, "99.90 percentile latency (ns)": 31730751, "Max latency (ns)": 42127269, "Mean latency (ns)": 20235021, "Min duration satisfied": "Yes", "Min latency (ns)": 18729001, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 49.38, "QPS w/o loadgen overhead": 49.42, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.282, "characteristics.good": 32141, "characteristics.power": 5.893500000000007, "characteristics.power.normalized_per_core": 0.04911250000000005, "characteristics.power.normalized_per_processor": 5.893500000000007, "characteristics.samples_per_second": 49.42, "characteristics.samples_per_second.normalized_per_core": 0.41183333333333333, "characteristics.samples_per_second.normalized_per_processor": 49.42, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 47.619, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "733a8f3d84b17ab9", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 38025970, "90.00 percentile latency (ns)": 45710216, "90th percentile latency (ns)": 45710216, "95.00 percentile latency (ns)": 52111799, "97.00 percentile latency (ns)": 52827270, "99.00 percentile latency (ns)": 55350512, "99.90 percentile latency (ns)": 61991122, "Max latency (ns)": 69816651, "Mean latency (ns)": 39671898, "Min duration satisfied": "Yes", "Min latency (ns)": 37362822, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 25.2, "QPS w/o loadgen overhead": 25.21, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.678, "characteristics.good": 36339, "characteristics.power": 6.16641666666666, "characteristics.power.normalized_per_core": 0.051386805555555505, "characteristics.power.normalized_per_processor": 6.16641666666666, "characteristics.samples_per_second": 25.21, "characteristics.samples_per_second.normalized_per_core": 0.21008333333333334, "characteristics.samples_per_second.normalized_per_processor": 25.21, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 26.3158, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "835c2d4ee14bbcaf", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 14525846, "90.00 percentile latency (ns)": 21036400, "90th percentile latency (ns)": 21036400, "95.00 percentile latency (ns)": 22611682, "97.00 percentile latency (ns)": 23389384, "99.00 percentile latency (ns)": 24452127, "99.90 percentile latency (ns)": 28268665, "Max latency (ns)": 37618865, "Mean latency (ns)": 15891947, "Min duration satisfied": "Yes", "Min latency (ns)": 13661268, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 62.86, "QPS w/o loadgen overhead": 62.92, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 65.004, "characteristics.good": 32502, "characteristics.power": 6.458399999999999, "characteristics.power.normalized_per_core": 0.05381999999999999, "characteristics.power.normalized_per_processor": 6.458399999999999, "characteristics.samples_per_second": 62.92, "characteristics.samples_per_second.normalized_per_core": 0.5243333333333333, "characteristics.samples_per_second.normalized_per_processor": 62.92, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 62.5, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "85949df9339f6608", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 51629603, "90.00 percentile latency (ns)": 62724501, "90th percentile latency (ns)": 62724501, "95.00 percentile latency (ns)": 69310010, "97.00 percentile latency (ns)": 71304535, "99.00 percentile latency (ns)": 74047822, "99.90 percentile latency (ns)": 80887195, "Max latency (ns)": 84392553, "Mean latency (ns)": 53821853, "Min duration satisfied": "Yes", "Min latency (ns)": 49553914, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 18.57, "QPS w/o loadgen overhead": 18.58, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.77, "characteristics.good": 35885, "characteristics.power": 6.251316666666681, "characteristics.power.normalized_per_core": 0.05209430555555568, "characteristics.power.normalized_per_processor": 6.251316666666681, "characteristics.samples_per_second": 18.58, "characteristics.samples_per_second.normalized_per_core": 0.15483333333333332, "characteristics.samples_per_second.normalized_per_processor": 18.58, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 16.9492, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "54b0a8b300e658ed", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 23948307, "90.00 percentile latency (ns)": 28996971, "90th percentile latency (ns)": 28996971, "95.00 percentile latency (ns)": 31888387, "97.00 percentile latency (ns)": 32659812, "99.00 percentile latency (ns)": 34251037, "99.90 percentile latency (ns)": 39551119, "Max latency (ns)": 51930986, "Mean latency (ns)": 24992624, "Min duration satisfied": "Yes", "Min latency (ns)": 23274212, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 39.99, "QPS w/o loadgen overhead": 40.01, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.75, "characteristics.good": 33375, "characteristics.power": 6.135133333333337, "characteristics.power.normalized_per_core": 0.051126111111111136, "characteristics.power.normalized_per_processor": 6.135133333333337, "characteristics.samples_per_second": 40.01, "characteristics.samples_per_second.normalized_per_core": 0.33341666666666664, "characteristics.samples_per_second.normalized_per_processor": 40.01, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 35.7143, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "e9914c199e635d59", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 6724553, "90.00 percentile latency (ns)": 7927586, "90th percentile latency (ns)": 7927586, "95.00 percentile latency (ns)": 8344321, "97.00 percentile latency (ns)": 8603526, "99.00 percentile latency (ns)": 9307261, "99.90 percentile latency (ns)": 12232265, "Max latency (ns)": 26436590, "Mean latency (ns)": 6954529, "Min duration satisfied": "Yes", "Min latency (ns)": 6159232, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 143.47, "QPS w/o loadgen overhead": 143.79, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 47.14, "characteristics.good": 23570, "characteristics.power": 5.69983333333332, "characteristics.power.normalized_per_core": 0.047498611111111, "characteristics.power.normalized_per_processor": 5.69983333333332, "characteristics.samples_per_second": 143.79, "characteristics.samples_per_second.normalized_per_core": 1.19825, "characteristics.samples_per_second.normalized_per_processor": 143.79, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "33a5c0abb04fe08c", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 11890369, "90.00 percentile latency (ns)": 14044344, "90th percentile latency (ns)": 14044344, "95.00 percentile latency (ns)": 15037725, "97.00 percentile latency (ns)": 15447644, "99.00 percentile latency (ns)": 16551900, "99.90 percentile latency (ns)": 19974142, "Max latency (ns)": 28708742, "Mean latency (ns)": 12345146, "Min duration satisfied": "Yes", "Min latency (ns)": 11382365, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 80.9, "QPS w/o loadgen overhead": 81.0, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.988, "characteristics.good": 28494, "characteristics.power": 5.761066666666684, "characteristics.power.normalized_per_core": 0.04800888888888903, "characteristics.power.normalized_per_processor": 5.761066666666684, "characteristics.samples_per_second": 81.0, "characteristics.samples_per_second.normalized_per_core": 0.675, "characteristics.samples_per_second.normalized_per_processor": 81.0, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "94f4b9edbc0160ee", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 13114836, "90.00 percentile latency (ns)": 16171374, "90th percentile latency (ns)": 16171374, "95.00 percentile latency (ns)": 18603539, "97.00 percentile latency (ns)": 19136934, "99.00 percentile latency (ns)": 20221407, "99.90 percentile latency (ns)": 23842729, "Max latency (ns)": 34048431, "Mean latency (ns)": 13740978, "Min duration satisfied": "Yes", "Min latency (ns)": 12397186, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 72.69, "QPS w/o loadgen overhead": 72.78, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.716, "characteristics.good": 31858, "characteristics.power": 6.548400000000007, "characteristics.power.normalized_per_core": 0.054570000000000056, "characteristics.power.normalized_per_processor": 6.548400000000007, "characteristics.samples_per_second": 72.78, "characteristics.samples_per_second.normalized_per_core": 0.6065, "characteristics.samples_per_second.normalized_per_processor": 72.78, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "48118271a00ac775", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 6590922, "90.00 percentile latency (ns)": 9579815, "90th percentile latency (ns)": 9579815, "95.00 percentile latency (ns)": 9841597, "97.00 percentile latency (ns)": 10022861, "99.00 percentile latency (ns)": 10582632, "99.90 percentile latency (ns)": 13540465, "Max latency (ns)": 30131882, "Mean latency (ns)": 7053799, "Min duration satisfied": "Yes", "Min latency (ns)": 6125369, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 141.44, "QPS w/o loadgen overhead": 141.77, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 55.51, "characteristics.good": 27755, "characteristics.power": 6.105366666666671, "characteristics.power.normalized_per_core": 0.05087805555555559, "characteristics.power.normalized_per_processor": 6.105366666666671, "characteristics.samples_per_second": 141.77, "characteristics.samples_per_second.normalized_per_core": 1.1814166666666668, "characteristics.samples_per_second.normalized_per_processor": 141.77, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ed14baea40f14d48", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 15966925, "90.00 percentile latency (ns)": 18556755, "90th percentile latency (ns)": 18556755, "95.00 percentile latency (ns)": 19811995, "97.00 percentile latency (ns)": 20260376, "99.00 percentile latency (ns)": 21669166, "99.90 percentile latency (ns)": 25443207, "Max latency (ns)": 30910570, "Mean latency (ns)": 16502016, "Min duration satisfied": "Yes", "Min latency (ns)": 15377842, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 60.54, "QPS w/o loadgen overhead": 60.6, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.322, "characteristics.good": 29661, "characteristics.power": 5.7438833333333275, "characteristics.power.normalized_per_core": 0.0478656944444444, "characteristics.power.normalized_per_processor": 5.7438833333333275, "characteristics.samples_per_second": 60.6, "characteristics.samples_per_second.normalized_per_core": 0.505, "characteristics.samples_per_second.normalized_per_processor": 60.6, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-armnn-v21.02-neon", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 55.5556, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "54661ecc92942afa", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 688589520, "90.00 percentile latency (ns)": 689064385, "90th percentile latency (ns)": 689064385, "95.00 percentile latency (ns)": 689366538, "97.00 percentile latency (ns)": 689661492, "99.00 percentile latency (ns)": 693607309, "99.90 percentile latency (ns)": 783312487, "Max latency (ns)": 800537515, "Mean latency (ns)": 689123663, "Min duration satisfied": "Yes", "Min latency (ns)": 687981346, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.45, "QPS w/o loadgen overhead": 1.45, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 78.564, "characteristics.good": 39282, "characteristics.samples_per_second": 1.45, "characteristics.samples_per_second.normalized_per_core": 0.012083333333333333, "characteristics.samples_per_second.normalized_per_processor": 1.45, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite4-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 1.44928, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "54758e146ecb8e25", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 72459133, "90.00 percentile latency (ns)": 72976454, "90th percentile latency (ns)": 72976454, "95.00 percentile latency (ns)": 73316699, "97.00 percentile latency (ns)": 73582700, "99.00 percentile latency (ns)": 74334804, "99.90 percentile latency (ns)": 129499303, "Max latency (ns)": 140231906, "Mean latency (ns)": 72676613, "Min duration satisfied": "Yes", "Min latency (ns)": 71011671, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 13.76, "QPS w/o loadgen overhead": 13.76, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.068, "characteristics.good": 37534, "characteristics.samples_per_second": 13.76, "characteristics.samples_per_second.normalized_per_core": 0.11466666666666667, "characteristics.samples_per_second.normalized_per_processor": 13.76, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite0-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 13.8889, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d32bcaacf6b87f7f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 223723938, "90.00 percentile latency (ns)": 223937514, "90th percentile latency (ns)": 223937514, "95.00 percentile latency (ns)": 224060724, "97.00 percentile latency (ns)": 224192025, "99.00 percentile latency (ns)": 224625382, "99.90 percentile latency (ns)": 301158938, "Max latency (ns)": 344373952, "Mean latency (ns)": 223905253, "Min duration satisfied": "Yes", "Min latency (ns)": 223397206, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 4.47, "QPS w/o loadgen overhead": 4.47, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.116, "characteristics.good": 37058, "characteristics.samples_per_second": 4.47, "characteristics.samples_per_second.normalized_per_core": 0.03725, "characteristics.samples_per_second.normalized_per_processor": 4.47, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite1-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 4.44444, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "14c35b02e2f9e66c", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 40741143, "90.00 percentile latency (ns)": 40933925, "90th percentile latency (ns)": 40933925, "95.00 percentile latency (ns)": 41080064, "97.00 percentile latency (ns)": 41374525, "99.00 percentile latency (ns)": 45124533, "99.90 percentile latency (ns)": 77756619, "Max latency (ns)": 89106764, "Mean latency (ns)": 41028545, "Min duration satisfied": "Yes", "Min latency (ns)": 40525154, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 24.36, "QPS w/o loadgen overhead": 24.37, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.894, "characteristics.good": 36447, "characteristics.samples_per_second": 24.37, "characteristics.samples_per_second.normalized_per_core": 0.20308333333333334, "characteristics.samples_per_second.normalized_per_processor": 24.37, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.8095, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "74f29eb4d2fba420", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 301629367, "90.00 percentile latency (ns)": 301883393, "90th percentile latency (ns)": 301883393, "95.00 percentile latency (ns)": 302056356, "97.00 percentile latency (ns)": 302245902, "99.00 percentile latency (ns)": 304060099, "99.90 percentile latency (ns)": 396441420, "Max latency (ns)": 414375285, "Mean latency (ns)": 301916042, "Min duration satisfied": "Yes", "Min latency (ns)": 301247026, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 3.31, "QPS w/o loadgen overhead": 3.31, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.116, "characteristics.good": 37058, "characteristics.samples_per_second": 3.31, "characteristics.samples_per_second.normalized_per_core": 0.027583333333333335, "characteristics.samples_per_second.normalized_per_processor": 3.31, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite2-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.31126, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c5c337040da84f7f", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 98048657, "90.00 percentile latency (ns)": 98761221, "90th percentile latency (ns)": 98761221, "95.00 percentile latency (ns)": 99158176, "97.00 percentile latency (ns)": 99482974, "99.00 percentile latency (ns)": 101143520, "99.90 percentile latency (ns)": 180534342, "Max latency (ns)": 187476269, "Mean latency (ns)": 98436375, "Min duration satisfied": "Yes", "Min latency (ns)": 97337847, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 10.16, "QPS w/o loadgen overhead": 10.16, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.642, "characteristics.good": 38321, "characteristics.samples_per_second": 10.16, "characteristics.samples_per_second.normalized_per_core": 0.08466666666666667, "characteristics.samples_per_second.normalized_per_processor": 10.16, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite1-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 10.2041, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "a49a407677e87072", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 15348052, "90.00 percentile latency (ns)": 15793137, "90th percentile latency (ns)": 15793137, "95.00 percentile latency (ns)": 18605378, "97.00 percentile latency (ns)": 22355329, "99.00 percentile latency (ns)": 24494121, "99.90 percentile latency (ns)": 25873699, "Max latency (ns)": 37848632, "Mean latency (ns)": 15792427, "Min duration satisfied": "Yes", "Min latency (ns)": 14906471, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 63.24, "QPS w/o loadgen overhead": 63.32, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.33, "characteristics.good": 31665, "characteristics.samples_per_second": 63.32, "characteristics.samples_per_second.normalized_per_core": 0.5276666666666666, "characteristics.samples_per_second.normalized_per_processor": 63.32, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 62.5, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "25ec0a9994bb3ba9", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 361745630, "90.00 percentile latency (ns)": 362824084, "90th percentile latency (ns)": 362824084, "95.00 percentile latency (ns)": 363577611, "97.00 percentile latency (ns)": 364416736, "99.00 percentile latency (ns)": 365534448, "99.90 percentile latency (ns)": 453861036, "Max latency (ns)": 516532787, "Mean latency (ns)": 362145779, "Min duration satisfied": "Yes", "Min latency (ns)": 360074226, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.76, "QPS w/o loadgen overhead": 2.76, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 80.676, "characteristics.good": 40338, "characteristics.samples_per_second": 2.76, "characteristics.samples_per_second.normalized_per_core": 0.023, "characteristics.samples_per_second.normalized_per_processor": 2.76, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite4-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.76243, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4cc742c78d89816b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 33372731, "90.00 percentile latency (ns)": 33506898, "90th percentile latency (ns)": 33506898, "95.00 percentile latency (ns)": 33584481, "97.00 percentile latency (ns)": 33686564, "99.00 percentile latency (ns)": 34287105, "99.90 percentile latency (ns)": 62396722, "Max latency (ns)": 77390696, "Mean latency (ns)": 33527760, "Min duration satisfied": "Yes", "Min latency (ns)": 33222814, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 29.81, "QPS w/o loadgen overhead": 29.83, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.33, "characteristics.good": 35165, "characteristics.samples_per_second": 29.83, "characteristics.samples_per_second.normalized_per_core": 0.24858333333333332, "characteristics.samples_per_second.normalized_per_processor": 29.83, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 29.4118, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "a3678d0b4524afab", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 15168394, "90.00 percentile latency (ns)": 15340768, "90th percentile latency (ns)": 15340768, "95.00 percentile latency (ns)": 16243475, "97.00 percentile latency (ns)": 19760971, "99.00 percentile latency (ns)": 26926627, "99.90 percentile latency (ns)": 29326166, "Max latency (ns)": 39957105, "Mean latency (ns)": 15542250, "Min duration satisfied": "Yes", "Min latency (ns)": 15039478, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 64.22, "QPS w/o loadgen overhead": 64.34, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.062, "characteristics.good": 32031, "characteristics.samples_per_second": 64.34, "characteristics.samples_per_second.normalized_per_core": 0.5361666666666667, "characteristics.samples_per_second.normalized_per_processor": 64.34, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 62.5, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "1307cb4838aa6231", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 139205226, "90.00 percentile latency (ns)": 139828766, "90th percentile latency (ns)": 139828766, "95.00 percentile latency (ns)": 140338927, "97.00 percentile latency (ns)": 140795678, "99.00 percentile latency (ns)": 142831158, "99.90 percentile latency (ns)": 233800751, "Max latency (ns)": 254391639, "Mean latency (ns)": 139657638, "Min duration satisfied": "Yes", "Min latency (ns)": 137502669, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.16, "QPS w/o loadgen overhead": 7.16, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 77.542, "characteristics.good": 38771, "characteristics.samples_per_second": 7.16, "characteristics.samples_per_second.normalized_per_core": 0.059666666666666666, "characteristics.samples_per_second.normalized_per_processor": 7.16, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite2-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 7.0922, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "5162dbd48f19cd67", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 45113192, "90.00 percentile latency (ns)": 46361814, "90th percentile latency (ns)": 46361814, "95.00 percentile latency (ns)": 51594304, "97.00 percentile latency (ns)": 55857604, "99.00 percentile latency (ns)": 79354819, "99.90 percentile latency (ns)": 91711255, "Max latency (ns)": 98287446, "Mean latency (ns)": 46313448, "Min duration satisfied": "Yes", "Min latency (ns)": 44591115, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 21.58, "QPS w/o loadgen overhead": 21.59, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.724, "characteristics.good": 36862, "characteristics.samples_per_second": 21.59, "characteristics.samples_per_second.normalized_per_core": 0.17991666666666667, "characteristics.samples_per_second.normalized_per_processor": 21.59, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 21.2766, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "2a95d63e515869c9", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 27840993, "90.00 percentile latency (ns)": 28723283, "90th percentile latency (ns)": 28723283, "95.00 percentile latency (ns)": 33609857, "97.00 percentile latency (ns)": 37818892, "99.00 percentile latency (ns)": 46791418, "99.90 percentile latency (ns)": 55139195, "Max latency (ns)": 60985643, "Mean latency (ns)": 28670695, "Min duration satisfied": "Yes", "Min latency (ns)": 27327661, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 34.85, "QPS w/o loadgen overhead": 34.88, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.336, "characteristics.good": 34168, "characteristics.samples_per_second": 34.88, "characteristics.samples_per_second.normalized_per_core": 0.2906666666666667, "characteristics.samples_per_second.normalized_per_processor": 34.88, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 34.4828, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "fba08f04eaebbd9e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 138083701, "90.00 percentile latency (ns)": 138268560, "90th percentile latency (ns)": 138268560, "95.00 percentile latency (ns)": 138376187, "97.00 percentile latency (ns)": 138494583, "99.00 percentile latency (ns)": 139334355, "99.90 percentile latency (ns)": 233064800, "Max latency (ns)": 241204705, "Mean latency (ns)": 138377900, "Min duration satisfied": "Yes", "Min latency (ns)": 137799898, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.23, "QPS w/o loadgen overhead": 7.23, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.254, "characteristics.good": 35627, "characteristics.samples_per_second": 7.23, "characteristics.samples_per_second.normalized_per_core": 0.060250000000000005, "characteristics.samples_per_second.normalized_per_processor": 7.23, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite0-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 7.19424, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "489dbb938b5f305f", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 209859952, "90.00 percentile latency (ns)": 210661185, "90th percentile latency (ns)": 210661185, "95.00 percentile latency (ns)": 211107195, "97.00 percentile latency (ns)": 211736573, "99.00 percentile latency (ns)": 213167779, "99.90 percentile latency (ns)": 326696120, "Max latency (ns)": 350033695, "Mean latency (ns)": 210189286, "Min duration satisfied": "Yes", "Min latency (ns)": 208051609, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 4.76, "QPS w/o loadgen overhead": 4.76, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 79.268, "characteristics.good": 39634, "characteristics.samples_per_second": 4.76, "characteristics.samples_per_second.normalized_per_core": 0.03966666666666666, "characteristics.samples_per_second.normalized_per_processor": 4.76, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite3-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 4.7619, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6a997e3a594fe2a3", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 24109753, "90.00 percentile latency (ns)": 24735684, "90th percentile latency (ns)": 24735684, "95.00 percentile latency (ns)": 28782290, "97.00 percentile latency (ns)": 33460043, "99.00 percentile latency (ns)": 39741967, "99.90 percentile latency (ns)": 44524453, "Max latency (ns)": 53896527, "Mean latency (ns)": 24792305, "Min duration satisfied": "Yes", "Min latency (ns)": 23703203, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 40.3, "QPS w/o loadgen overhead": 40.34, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.216, "characteristics.good": 33108, "characteristics.samples_per_second": 40.34, "characteristics.samples_per_second.normalized_per_core": 0.33616666666666667, "characteristics.samples_per_second.normalized_per_processor": 40.34, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "0886a570cc455d96", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 414238981, "90.00 percentile latency (ns)": 414618820, "90th percentile latency (ns)": 414618820, "95.00 percentile latency (ns)": 414882080, "97.00 percentile latency (ns)": 415104228, "99.00 percentile latency (ns)": 418652669, "99.90 percentile latency (ns)": 491298325, "Max latency (ns)": 492367215, "Mean latency (ns)": 414563305, "Min duration satisfied": "Yes", "Min latency (ns)": 413679865, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.41, "QPS w/o loadgen overhead": 2.41, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.348, "characteristics.good": 38174, "characteristics.samples_per_second": 2.41, "characteristics.samples_per_second.normalized_per_core": 0.020083333333333335, "characteristics.samples_per_second.normalized_per_processor": 2.41, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite3-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.39808, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "7cca99b80f7340d6", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 57709161, "90.00 percentile latency (ns)": 58421697, "90th percentile latency (ns)": 58421697, "95.00 percentile latency (ns)": 58727357, "97.00 percentile latency (ns)": 59152038, "99.00 percentile latency (ns)": 82368088, "99.90 percentile latency (ns)": 111624627, "Max latency (ns)": 121880159, "Mean latency (ns)": 58334815, "Min duration satisfied": "Yes", "Min latency (ns)": 57261462, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.14, "QPS w/o loadgen overhead": 17.14, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.76, "characteristics.good": 37880, "characteristics.samples_per_second": 17.14, "characteristics.samples_per_second.normalized_per_core": 0.14283333333333334, "characteristics.samples_per_second.normalized_per_processor": 17.14, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 16.9492, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "eabaf24849f615f2", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 38988791, "90.00 percentile latency (ns)": 40207956, "90th percentile latency (ns)": 40207956, "95.00 percentile latency (ns)": 47062410, "97.00 percentile latency (ns)": 53708931, "99.00 percentile latency (ns)": 68143509, "99.90 percentile latency (ns)": 74824377, "Max latency (ns)": 79010661, "Mean latency (ns)": 40167476, "Min duration satisfied": "Yes", "Min latency (ns)": 38315356, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 24.88, "QPS w/o loadgen overhead": 24.9, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.678, "characteristics.good": 36339, "characteristics.samples_per_second": 24.9, "characteristics.samples_per_second.normalized_per_core": 0.2075, "characteristics.samples_per_second.normalized_per_processor": 24.9, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 24.3902, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "b9649f28037889da", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 582452195, "90.00 percentile latency (ns)": 587150887, "90th percentile latency (ns)": 587150887, "95.00 percentile latency (ns)": 591833086, "97.00 percentile latency (ns)": 593263107, "99.00 percentile latency (ns)": 594836677, "99.90 percentile latency (ns)": 596362554, "Max latency (ns)": 597205340, "Mean latency (ns)": 581859903, "Min duration satisfied": "Yes", "Min latency (ns)": 574840061, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.72, "QPS w/o loadgen overhead": 1.72, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 78.536, "characteristics.good": 39268, "characteristics.samples_per_second": 1.72, "characteristics.samples_per_second.normalized_per_core": 1.72, "characteristics.samples_per_second.normalized_per_processor": 1.72, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite4-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 1.72117, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d6b88ff34ad5d292", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 92077203, "90.00 percentile latency (ns)": 92771952, "90th percentile latency (ns)": 92771952, "95.00 percentile latency (ns)": 93167743, "97.00 percentile latency (ns)": 93535242, "99.00 percentile latency (ns)": 98509314, "99.90 percentile latency (ns)": 112348865, "Max latency (ns)": 124231922, "Mean latency (ns)": 92277982, "Min duration satisfied": "Yes", "Min latency (ns)": 83433724, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 10.82, "QPS w/o loadgen overhead": 10.84, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.068, "characteristics.good": 37534, "characteristics.samples_per_second": 10.84, "characteristics.samples_per_second.normalized_per_core": 10.84, "characteristics.samples_per_second.normalized_per_processor": 10.84, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite0-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 10.7527, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "55f460f82c9e3df9", "use_accelerator": true, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 171351700, "90.00 percentile latency (ns)": 179311509, "90th percentile latency (ns)": 179311509, "95.00 percentile latency (ns)": 180673320, "97.00 percentile latency (ns)": 181214994, "99.00 percentile latency (ns)": 185328367, "99.90 percentile latency (ns)": 186856061, "Max latency (ns)": 193018412, "Mean latency (ns)": 172779793, "Min duration satisfied": "Yes", "Min latency (ns)": 170227294, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 5.78, "QPS w/o loadgen overhead": 5.79, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.138, "characteristics.good": 37069, "characteristics.samples_per_second": 5.79, "characteristics.samples_per_second.normalized_per_core": 5.79, "characteristics.samples_per_second.normalized_per_processor": 5.79, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite1-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 5.74713, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "9e20bdaa405c9572", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 61462501, "90.00 percentile latency (ns)": 62036500, "90th percentile latency (ns)": 62036500, "95.00 percentile latency (ns)": 62339604, "97.00 percentile latency (ns)": 62889374, "99.00 percentile latency (ns)": 69332860, "99.90 percentile latency (ns)": 76950550, "Max latency (ns)": 82938745, "Mean latency (ns)": 61533094, "Min duration satisfied": "Yes", "Min latency (ns)": 59900628, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.23, "QPS w/o loadgen overhead": 16.25, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.892, "characteristics.good": 36446, "characteristics.samples_per_second": 16.25, "characteristics.samples_per_second.normalized_per_core": 16.25, "characteristics.samples_per_second.normalized_per_processor": 16.25, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 16.129, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6fa79a3c14c319a1", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 242964773, "90.00 percentile latency (ns)": 250363761, "90th percentile latency (ns)": 250363761, "95.00 percentile latency (ns)": 251922711, "97.00 percentile latency (ns)": 252718379, "99.00 percentile latency (ns)": 255873912, "99.90 percentile latency (ns)": 259843487, "Max latency (ns)": 262103022, "Mean latency (ns)": 244151395, "Min duration satisfied": "Yes", "Min latency (ns)": 238520954, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 4.09, "QPS w/o loadgen overhead": 4.1, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.13, "characteristics.good": 37065, "characteristics.samples_per_second": 4.1, "characteristics.samples_per_second.normalized_per_core": 4.1, "characteristics.samples_per_second.normalized_per_processor": 4.1, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite2-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 4.08163, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "f27f66e13c0d452f", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 133440668, "90.00 percentile latency (ns)": 134415082, "90th percentile latency (ns)": 134415082, "95.00 percentile latency (ns)": 134876234, "97.00 percentile latency (ns)": 135531942, "99.00 percentile latency (ns)": 140712298, "99.90 percentile latency (ns)": 148119463, "Max latency (ns)": 157122873, "Mean latency (ns)": 133627930, "Min duration satisfied": "Yes", "Min latency (ns)": 123842552, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.48, "QPS w/o loadgen overhead": 7.48, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.642, "characteristics.good": 38321, "characteristics.samples_per_second": 7.48, "characteristics.samples_per_second.normalized_per_core": 7.48, "characteristics.samples_per_second.normalized_per_processor": 7.48, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite1-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 7.46269, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6496f034c9413fbf", "use_accelerator": true, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 23673404, "90.00 percentile latency (ns)": 24100401, "90th percentile latency (ns)": 24100401, "95.00 percentile latency (ns)": 24410151, "97.00 percentile latency (ns)": 25348442, "99.00 percentile latency (ns)": 31527979, "99.90 percentile latency (ns)": 37282855, "Max latency (ns)": 43558932, "Mean latency (ns)": 23907616, "Min duration satisfied": "Yes", "Min latency (ns)": 23083069, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 41.78, "QPS w/o loadgen overhead": 41.83, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.33, "characteristics.good": 31665, "characteristics.samples_per_second": 41.83, "characteristics.samples_per_second.normalized_per_core": 41.83, "characteristics.samples_per_second.normalized_per_processor": 41.83, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 41.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "cf26d0b3431b86c2", "use_accelerator": true, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 479143617, "90.00 percentile latency (ns)": 481495572, "90th percentile latency (ns)": 481495572, "95.00 percentile latency (ns)": 482713587, "97.00 percentile latency (ns)": 484626298, "99.00 percentile latency (ns)": 489298256, "99.90 percentile latency (ns)": 505017013, "Max latency (ns)": 507408394, "Mean latency (ns)": 479205108, "Min duration satisfied": "Yes", "Min latency (ns)": 463269958, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.09, "QPS w/o loadgen overhead": 2.09, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 80.676, "characteristics.good": 40338, "characteristics.samples_per_second": 2.09, "characteristics.samples_per_second.normalized_per_core": 2.09, "characteristics.samples_per_second.normalized_per_processor": 2.09, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite4-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.079, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "b049dc89fbd1f172", "use_accelerator": true, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 52647317, "90.00 percentile latency (ns)": 53112259, "90th percentile latency (ns)": 53112259, "95.00 percentile latency (ns)": 53369547, "97.00 percentile latency (ns)": 53730880, "99.00 percentile latency (ns)": 61375523, "99.90 percentile latency (ns)": 67803327, "Max latency (ns)": 73487869, "Mean latency (ns)": 52855180, "Min duration satisfied": "Yes", "Min latency (ns)": 51622451, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 18.87, "QPS w/o loadgen overhead": 18.92, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.328, "characteristics.good": 35164, "characteristics.samples_per_second": 18.92, "characteristics.samples_per_second.normalized_per_core": 18.92, "characteristics.samples_per_second.normalized_per_processor": 18.92, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.5185, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "35ba6ce9ad8e490c", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 25742692, "90.00 percentile latency (ns)": 26109630, "90th percentile latency (ns)": 26109630, "95.00 percentile latency (ns)": 26301544, "97.00 percentile latency (ns)": 26639591, "99.00 percentile latency (ns)": 31321132, "99.90 percentile latency (ns)": 38769400, "Max latency (ns)": 44727284, "Mean latency (ns)": 25892448, "Min duration satisfied": "Yes", "Min latency (ns)": 25105151, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 38.56, "QPS w/o loadgen overhead": 38.62, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.03, "characteristics.good": 32015, "characteristics.samples_per_second": 38.62, "characteristics.samples_per_second.normalized_per_core": 38.62, "characteristics.samples_per_second.normalized_per_processor": 38.62, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 38.4615, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "34a3e7b9a232c6b9", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 179136609, "90.00 percentile latency (ns)": 180590856, "90th percentile latency (ns)": 180590856, "95.00 percentile latency (ns)": 184481093, "97.00 percentile latency (ns)": 187059125, "99.00 percentile latency (ns)": 189869041, "99.90 percentile latency (ns)": 202634961, "Max latency (ns)": 209420874, "Mean latency (ns)": 179651825, "Min duration satisfied": "Yes", "Min latency (ns)": 167788053, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 5.56, "QPS w/o loadgen overhead": 5.57, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 77.542, "characteristics.good": 38771, "characteristics.samples_per_second": 5.57, "characteristics.samples_per_second.normalized_per_core": 5.57, "characteristics.samples_per_second.normalized_per_processor": 5.57, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite2-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 5.55556, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "a133464d994432ee", "use_accelerator": true, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 57573761, "90.00 percentile latency (ns)": 58055590, "90th percentile latency (ns)": 58055590, "95.00 percentile latency (ns)": 58311093, "97.00 percentile latency (ns)": 58517591, "99.00 percentile latency (ns)": 59307710, "99.90 percentile latency (ns)": 71634408, "Max latency (ns)": 81538722, "Mean latency (ns)": 55470660, "Min duration satisfied": "Yes", "Min latency (ns)": 49126229, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 18.0, "QPS w/o loadgen overhead": 18.03, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.724, "characteristics.good": 36862, "characteristics.samples_per_second": 18.03, "characteristics.samples_per_second.normalized_per_core": 18.03, "characteristics.samples_per_second.normalized_per_processor": 18.03, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 19.2308, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "024ba7a71ca8b369", "use_accelerator": true, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 26218791, "90.00 percentile latency (ns)": 26676717, "90th percentile latency (ns)": 26676717, "95.00 percentile latency (ns)": 26966907, "97.00 percentile latency (ns)": 27291254, "99.00 percentile latency (ns)": 29219156, "99.90 percentile latency (ns)": 40298115, "Max latency (ns)": 45181780, "Mean latency (ns)": 26390186, "Min duration satisfied": "Yes", "Min latency (ns)": 25638957, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 37.86, "QPS w/o loadgen overhead": 37.89, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.336, "characteristics.good": 34168, "characteristics.samples_per_second": 37.89, "characteristics.samples_per_second.normalized_per_core": 37.89, "characteristics.samples_per_second.normalized_per_processor": 37.89, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 37.037, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "417b5d7a7281bbd6", "use_accelerator": true, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 116801347, "90.00 percentile latency (ns)": 123030208, "90th percentile latency (ns)": 123030208, "95.00 percentile latency (ns)": 125638205, "97.00 percentile latency (ns)": 126176857, "99.00 percentile latency (ns)": 128755492, "99.90 percentile latency (ns)": 129642808, "Max latency (ns)": 130568458, "Mean latency (ns)": 117993234, "Min duration satisfied": "Yes", "Min latency (ns)": 115694068, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 8.47, "QPS w/o loadgen overhead": 8.48, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.288, "characteristics.good": 35644, "characteristics.samples_per_second": 8.48, "characteristics.samples_per_second.normalized_per_core": 8.48, "characteristics.samples_per_second.normalized_per_processor": 8.48, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite0-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 8.40336, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "f433f86de0b83595", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 265083697, "90.00 percentile latency (ns)": 270864146, "90th percentile latency (ns)": 270864146, "95.00 percentile latency (ns)": 273176295, "97.00 percentile latency (ns)": 273820128, "99.00 percentile latency (ns)": 279520044, "99.90 percentile latency (ns)": 289131953, "Max latency (ns)": 296748355, "Mean latency (ns)": 265928345, "Min duration satisfied": "Yes", "Min latency (ns)": 251569204, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 3.76, "QPS w/o loadgen overhead": 3.76, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 79.268, "characteristics.good": 39634, "characteristics.samples_per_second": 3.76, "characteristics.samples_per_second.normalized_per_core": 3.76, "characteristics.samples_per_second.normalized_per_processor": 3.76, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite3-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.74532, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "0e434fbffce21d1a", "use_accelerator": true, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 23553181, "90.00 percentile latency (ns)": 23995915, "90th percentile latency (ns)": 23995915, "95.00 percentile latency (ns)": 24275616, "97.00 percentile latency (ns)": 24552134, "99.00 percentile latency (ns)": 26091830, "99.90 percentile latency (ns)": 37893200, "Max latency (ns)": 44037752, "Mean latency (ns)": 23719199, "Min duration satisfied": "Yes", "Min latency (ns)": 23031677, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 42.11, "QPS w/o loadgen overhead": 42.16, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.216, "characteristics.good": 33108, "characteristics.samples_per_second": 42.16, "characteristics.samples_per_second.normalized_per_core": 42.16, "characteristics.samples_per_second.normalized_per_processor": 42.16, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d80599db71430651", "use_accelerator": true, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 349431038, "90.00 percentile latency (ns)": 357869938, "90th percentile latency (ns)": 357869938, "95.00 percentile latency (ns)": 358950832, "97.00 percentile latency (ns)": 360467229, "99.00 percentile latency (ns)": 363381166, "99.90 percentile latency (ns)": 369977208, "Max latency (ns)": 374280230, "Mean latency (ns)": 351414817, "Min duration satisfied": "Yes", "Min latency (ns)": 347785808, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.85, "QPS w/o loadgen overhead": 2.85, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.338, "characteristics.good": 38169, "characteristics.samples_per_second": 2.85, "characteristics.samples_per_second.normalized_per_core": 2.85, "characteristics.samples_per_second.normalized_per_processor": 2.85, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite3-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.84091, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "40e384f75fb3f738", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 67700388, "90.00 percentile latency (ns)": 68163859, "90th percentile latency (ns)": 68163859, "95.00 percentile latency (ns)": 68414986, "97.00 percentile latency (ns)": 68685332, "99.00 percentile latency (ns)": 69885546, "99.90 percentile latency (ns)": 78403087, "Max latency (ns)": 84725543, "Mean latency (ns)": 67100150, "Min duration satisfied": "Yes", "Min latency (ns)": 59322575, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 14.88, "QPS w/o loadgen overhead": 14.9, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.76, "characteristics.good": 37880, "characteristics.samples_per_second": 14.9, "characteristics.samples_per_second.normalized_per_core": 14.9, "characteristics.samples_per_second.normalized_per_processor": 14.9, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.9254, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "574d9e9837206954", "use_accelerator": true, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 67979752, "90.00 percentile latency (ns)": 68638354, "90th percentile latency (ns)": 68638354, "95.00 percentile latency (ns)": 69341562, "97.00 percentile latency (ns)": 76273872, "99.00 percentile latency (ns)": 77184159, "99.90 percentile latency (ns)": 86105627, "Max latency (ns)": 92580615, "Mean latency (ns)": 68378154, "Min duration satisfied": "Yes", "Min latency (ns)": 59628782, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 14.58, "QPS w/o loadgen overhead": 14.62, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.678, "characteristics.good": 36339, "characteristics.samples_per_second": 14.62, "characteristics.samples_per_second.normalized_per_core": 14.62, "characteristics.samples_per_second.normalized_per_processor": 14.62, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.4928, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "2de580f126d74614", "use_accelerator": true, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 8983008, "90.00 percentile latency (ns)": 11994254, "90th percentile latency (ns)": 11994254, "95.00 percentile latency (ns)": 13057511, "97.00 percentile latency (ns)": 13861495, "99.00 percentile latency (ns)": 15999774, "99.90 percentile latency (ns)": 22223610, "Max latency (ns)": 31159683, "Mean latency (ns)": 9267563, "Min duration satisfied": "Yes", "Min latency (ns)": 5514868, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 107.73, "QPS w/o loadgen overhead": 107.9, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.242, "characteristics.good": 34121, "characteristics.power": 15.904450000000002, "characteristics.power.normalized_per_core": 0.13253708333333336, "characteristics.power.normalized_per_processor": 15.904450000000002, "characteristics.samples_per_second": 107.9, "characteristics.samples_per_second.normalized_per_core": 0.8991666666666667, "characteristics.samples_per_second.normalized_per_processor": 107.9, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "9d94337a35f7a403", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 11663442, "90.00 percentile latency (ns)": 16997262, "90th percentile latency (ns)": 16997262, "95.00 percentile latency (ns)": 19047518, "97.00 percentile latency (ns)": 20359298, "99.00 percentile latency (ns)": 23394958, "99.90 percentile latency (ns)": 29416527, "Max latency (ns)": 40216984, "Mean latency (ns)": 12111581, "Min duration satisfied": "Yes", "Min latency (ns)": 7550552, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 82.46, "QPS w/o loadgen overhead": 82.57, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.738, "characteristics.good": 35369, "characteristics.power": 15.522983333333311, "characteristics.power.normalized_per_core": 0.12935819444444427, "characteristics.power.normalized_per_processor": 15.522983333333311, "characteristics.samples_per_second": 82.57, "characteristics.samples_per_second.normalized_per_core": 0.6880833333333333, "characteristics.samples_per_second.normalized_per_processor": 82.57, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "a9dfebc64c7a8b42", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 151132999, "90.00 percentile latency (ns)": 170537284, "90th percentile latency (ns)": 170537284, "95.00 percentile latency (ns)": 180100665, "97.00 percentile latency (ns)": 185350585, "99.00 percentile latency (ns)": 202184917, "99.90 percentile latency (ns)": 237026307, "Max latency (ns)": 252815561, "Mean latency (ns)": 153860936, "Min duration satisfied": "Yes", "Min latency (ns)": 136330219, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 6.5, "QPS w/o loadgen overhead": 6.5, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 78.564, "characteristics.good": 39282, "characteristics.power": 20.195099999999996, "characteristics.power.normalized_per_core": 0.16829249999999998, "characteristics.power.normalized_per_processor": 20.195099999999996, "characteristics.samples_per_second": 6.5, "characteristics.samples_per_second.normalized_per_core": 0.05416666666666667, "characteristics.samples_per_second.normalized_per_processor": 6.5, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite4-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "414014c51bc2c8f5", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 13266433, "90.00 percentile latency (ns)": 18690818, "90th percentile latency (ns)": 18690818, "95.00 percentile latency (ns)": 20562766, "97.00 percentile latency (ns)": 21861347, "99.00 percentile latency (ns)": 25026121, "99.90 percentile latency (ns)": 30375269, "Max latency (ns)": 51035225, "Mean latency (ns)": 13598318, "Min duration satisfied": "Yes", "Min latency (ns)": 8640547, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 73.47, "QPS w/o loadgen overhead": 73.54, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.65, "characteristics.good": 31825, "characteristics.power": 13.185566666666666, "characteristics.power.normalized_per_core": 0.10987972222222223, "characteristics.power.normalized_per_processor": 13.185566666666666, "characteristics.samples_per_second": 73.54, "characteristics.samples_per_second.normalized_per_core": 0.6128333333333333, "characteristics.samples_per_second.normalized_per_processor": 73.54, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c7de4583aed4a068", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 6490698, "90.00 percentile latency (ns)": 11561178, "90th percentile latency (ns)": 11561178, "95.00 percentile latency (ns)": 12848655, "97.00 percentile latency (ns)": 13530123, "99.00 percentile latency (ns)": 14764126, "99.90 percentile latency (ns)": 21643607, "Max latency (ns)": 44737931, "Mean latency (ns)": 7327035, "Min duration satisfied": "Yes", "Min latency (ns)": 3748058, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 136.13, "QPS w/o loadgen overhead": 136.48, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 52.172, "characteristics.good": 26086, "characteristics.power": 10.690633333333333, "characteristics.power.normalized_per_core": 0.0890886111111111, "characteristics.power.normalized_per_processor": 10.690633333333333, "characteristics.samples_per_second": 136.48, "characteristics.samples_per_second.normalized_per_core": 1.1373333333333333, "characteristics.samples_per_second.normalized_per_processor": 136.48, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c454057ae9101e06", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 7679081, "90.00 percentile latency (ns)": 14015577, "90th percentile latency (ns)": 14015577, "95.00 percentile latency (ns)": 15201901, "97.00 percentile latency (ns)": 15815685, "99.00 percentile latency (ns)": 17064379, "99.90 percentile latency (ns)": 24203534, "Max latency (ns)": 33243178, "Mean latency (ns)": 8555816, "Min duration satisfied": "Yes", "Min latency (ns)": 4101200, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 116.59, "QPS w/o loadgen overhead": 116.88, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.402, "characteristics.good": 29201, "characteristics.power": 10.538999999999993, "characteristics.power.normalized_per_core": 0.08782499999999994, "characteristics.power.normalized_per_processor": 10.538999999999993, "characteristics.samples_per_second": 116.88, "characteristics.samples_per_second.normalized_per_core": 0.974, "characteristics.samples_per_second.normalized_per_processor": 116.88, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "073a0752327442fb", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 10659060, "90.00 percentile latency (ns)": 17533691, "90th percentile latency (ns)": 17533691, "95.00 percentile latency (ns)": 19275623, "97.00 percentile latency (ns)": 20299732, "99.00 percentile latency (ns)": 22130854, "99.90 percentile latency (ns)": 28602887, "Max latency (ns)": 39287664, "Mean latency (ns)": 11407188, "Min duration satisfied": "Yes", "Min latency (ns)": 5671447, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 87.5, "QPS w/o loadgen overhead": 87.66, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 60.976, "characteristics.good": 30488, "characteristics.power": 11.113483333333331, "characteristics.power.normalized_per_core": 0.0926123611111111, "characteristics.power.normalized_per_processor": 11.113483333333331, "characteristics.samples_per_second": 87.66, "characteristics.samples_per_second.normalized_per_core": 0.7304999999999999, "characteristics.samples_per_second.normalized_per_processor": 87.66, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "a4b4eac9ac59ebfd", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 6218182, "90.00 percentile latency (ns)": 10259121, "90th percentile latency (ns)": 10259121, "95.00 percentile latency (ns)": 11311997, "97.00 percentile latency (ns)": 11898963, "99.00 percentile latency (ns)": 12937659, "99.90 percentile latency (ns)": 18460456, "Max latency (ns)": 29604754, "Mean latency (ns)": 6655081, "Min duration satisfied": "Yes", "Min latency (ns)": 3264424, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 149.94, "QPS w/o loadgen overhead": 150.26, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 48.114, "characteristics.good": 24057, "characteristics.power": 10.323533333333325, "characteristics.power.normalized_per_core": 0.08602944444444437, "characteristics.power.normalized_per_processor": 10.323533333333325, "characteristics.samples_per_second": 150.26, "characteristics.samples_per_second.normalized_per_core": 1.2521666666666667, "characteristics.samples_per_second.normalized_per_processor": 150.26, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "bfa8fb84a40b054b", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 16872201, "90.00 percentile latency (ns)": 19510264, "90th percentile latency (ns)": 19510264, "95.00 percentile latency (ns)": 21136758, "97.00 percentile latency (ns)": 22854075, "99.00 percentile latency (ns)": 26325928, "99.90 percentile latency (ns)": 32059952, "Max latency (ns)": 39204083, "Mean latency (ns)": 16449028, "Min duration satisfied": "Yes", "Min latency (ns)": 10445125, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 60.73, "QPS w/o loadgen overhead": 60.79, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.068, "characteristics.good": 37534, "characteristics.power": 17.06233333333333, "characteristics.power.normalized_per_core": 0.1421861111111111, "characteristics.power.normalized_per_processor": 17.06233333333333, "characteristics.samples_per_second": 60.79, "characteristics.samples_per_second.normalized_per_core": 0.5065833333333333, "characteristics.samples_per_second.normalized_per_processor": 60.79, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite0-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "919d4ae79864d329", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 6287822, "90.00 percentile latency (ns)": 10529060, "90th percentile latency (ns)": 10529060, "95.00 percentile latency (ns)": 11745112, "97.00 percentile latency (ns)": 12522363, "99.00 percentile latency (ns)": 14002234, "99.90 percentile latency (ns)": 18996719, "Max latency (ns)": 27685098, "Mean latency (ns)": 6752544, "Min duration satisfied": "Yes", "Min latency (ns)": 3771490, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 147.84, "QPS w/o loadgen overhead": 148.09, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 50.164, "characteristics.good": 25082, "characteristics.power": 11.34948333333333, "characteristics.power.normalized_per_core": 0.09457902777777775, "characteristics.power.normalized_per_processor": 11.34948333333333, "characteristics.samples_per_second": 148.09, "characteristics.samples_per_second.normalized_per_core": 1.2340833333333334, "characteristics.samples_per_second.normalized_per_processor": 148.09, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "f4b1da85f901a40a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 5407848, "90.00 percentile latency (ns)": 10552771, "90th percentile latency (ns)": 10552771, "95.00 percentile latency (ns)": 11689076, "97.00 percentile latency (ns)": 12375346, "99.00 percentile latency (ns)": 13604422, "99.90 percentile latency (ns)": 19308315, "Max latency (ns)": 28866964, "Mean latency (ns)": 6132029, "Min duration satisfied": "Yes", "Min latency (ns)": 3134662, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 162.65, "QPS w/o loadgen overhead": 163.08, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 60.842, "characteristics.good": 30421, "characteristics.power": 11.847449999999997, "characteristics.power.normalized_per_core": 0.09872874999999998, "characteristics.power.normalized_per_processor": 11.847449999999997, "characteristics.samples_per_second": 163.08, "characteristics.samples_per_second.normalized_per_core": 1.3590000000000002, "characteristics.samples_per_second.normalized_per_processor": 163.08, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "7cd149ce90263484", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 10641061, "90.00 percentile latency (ns)": 16721840, "90th percentile latency (ns)": 16721840, "95.00 percentile latency (ns)": 18875246, "97.00 percentile latency (ns)": 20416596, "99.00 percentile latency (ns)": 23538085, "99.90 percentile latency (ns)": 29148005, "Max latency (ns)": 37968438, "Mean latency (ns)": 11194607, "Min duration satisfied": "Yes", "Min latency (ns)": 6208791, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 89.19, "QPS w/o loadgen overhead": 89.33, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.016, "characteristics.good": 34508, "characteristics.power": 13.982216666666654, "characteristics.power.normalized_per_core": 0.11651847222222211, "characteristics.power.normalized_per_processor": 13.982216666666654, "characteristics.samples_per_second": 89.33, "characteristics.samples_per_second.normalized_per_core": 0.7444166666666666, "characteristics.samples_per_second.normalized_per_processor": 89.33, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ee89c953925fdaa9", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 43286985, "90.00 percentile latency (ns)": 54647994, "90th percentile latency (ns)": 54647994, "95.00 percentile latency (ns)": 59504815, "97.00 percentile latency (ns)": 64602221, "99.00 percentile latency (ns)": 80422825, "99.90 percentile latency (ns)": 93267704, "Max latency (ns)": 104490340, "Mean latency (ns)": 46072718, "Min duration satisfied": "Yes", "Min latency (ns)": 41317329, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 21.7, "QPS w/o loadgen overhead": 21.7, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.116, "characteristics.good": 37058, "characteristics.power": 20.884283333333304, "characteristics.power.normalized_per_core": 0.1740356944444442, "characteristics.power.normalized_per_processor": 20.884283333333304, "characteristics.samples_per_second": 21.7, "characteristics.samples_per_second.normalized_per_core": 0.18083333333333332, "characteristics.samples_per_second.normalized_per_processor": 21.7, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite1-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "fe28c78973dc7e0b", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 13113534, "90.00 percentile latency (ns)": 20521308, "90th percentile latency (ns)": 20521308, "95.00 percentile latency (ns)": 22677864, "97.00 percentile latency (ns)": 24033782, "99.00 percentile latency (ns)": 26206104, "99.90 percentile latency (ns)": 30798954, "Max latency (ns)": 37852900, "Mean latency (ns)": 13673921, "Min duration satisfied": "Yes", "Min latency (ns)": 7596848, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 73.04, "QPS w/o loadgen overhead": 73.13, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.948, "characteristics.good": 30974, "characteristics.power": 11.311133333333323, "characteristics.power.normalized_per_core": 0.09425944444444437, "characteristics.power.normalized_per_processor": 11.311133333333323, "characteristics.samples_per_second": 73.13, "characteristics.samples_per_second.normalized_per_core": 0.6094166666666666, "characteristics.samples_per_second.normalized_per_processor": 73.13, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "59baa9cbd4efc9df", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 11732770, "90.00 percentile latency (ns)": 13838846, "90th percentile latency (ns)": 13838846, "95.00 percentile latency (ns)": 14858253, "97.00 percentile latency (ns)": 15696081, "99.00 percentile latency (ns)": 18067836, "99.90 percentile latency (ns)": 24159106, "Max latency (ns)": 38807562, "Mean latency (ns)": 11644468, "Min duration satisfied": "Yes", "Min latency (ns)": 7145915, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 85.8, "QPS w/o loadgen overhead": 85.88, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 62.264, "characteristics.good": 31132, "characteristics.power": 13.861950000000006, "characteristics.power.normalized_per_core": 0.11551625000000004, "characteristics.power.normalized_per_processor": 13.861950000000006, "characteristics.samples_per_second": 85.88, "characteristics.samples_per_second.normalized_per_core": 0.7156666666666667, "characteristics.samples_per_second.normalized_per_processor": 85.88, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "b5bac7246debb384", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 8792155, "90.00 percentile latency (ns)": 14268680, "90th percentile latency (ns)": 14268680, "95.00 percentile latency (ns)": 15791272, "97.00 percentile latency (ns)": 16606587, "99.00 percentile latency (ns)": 17911942, "99.90 percentile latency (ns)": 24310025, "Max latency (ns)": 33402072, "Mean latency (ns)": 9513778, "Min duration satisfied": "Yes", "Min latency (ns)": 5150303, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 104.93, "QPS w/o loadgen overhead": 105.11, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.19, "characteristics.good": 24595, "characteristics.power": 10.347216666666656, "characteristics.power.normalized_per_core": 0.08622680555555547, "characteristics.power.normalized_per_processor": 10.347216666666656, "characteristics.samples_per_second": 105.11, "characteristics.samples_per_second.normalized_per_core": 0.8759166666666667, "characteristics.samples_per_second.normalized_per_processor": 105.11, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "34c2f460aa644797", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 23532464, "90.00 percentile latency (ns)": 36884210, "90th percentile latency (ns)": 36884210, "95.00 percentile latency (ns)": 39973615, "97.00 percentile latency (ns)": 41542425, "99.00 percentile latency (ns)": 46135795, "99.90 percentile latency (ns)": 52699470, "Max latency (ns)": 61503620, "Mean latency (ns)": 26099460, "Min duration satisfied": "Yes", "Min latency (ns)": 22323514, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 38.29, "QPS w/o loadgen overhead": 38.31, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.556, "characteristics.good": 36778, "characteristics.power": 17.09816666666668, "characteristics.power.normalized_per_core": 0.14248472222222233, "characteristics.power.normalized_per_processor": 17.09816666666668, "characteristics.samples_per_second": 38.31, "characteristics.samples_per_second.normalized_per_core": 0.31925000000000003, "characteristics.samples_per_second.normalized_per_processor": 38.31, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.4-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.4-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_140.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 37.037, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "41de855740741242", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 18198831, "90.00 percentile latency (ns)": 21170270, "90th percentile latency (ns)": 21170270, "95.00 percentile latency (ns)": 29048994, "97.00 percentile latency (ns)": 30847223, "99.00 percentile latency (ns)": 33920489, "99.90 percentile latency (ns)": 40312491, "Max latency (ns)": 50666597, "Mean latency (ns)": 19235021, "Min duration satisfied": "Yes", "Min latency (ns)": 17097826, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 51.96, "QPS w/o loadgen overhead": 51.99, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.764, "characteristics.good": 35382, "characteristics.power": 17.567966666666674, "characteristics.power.normalized_per_core": 0.14639972222222228, "characteristics.power.normalized_per_processor": 17.567966666666674, "characteristics.samples_per_second": 51.99, "characteristics.samples_per_second.normalized_per_core": 0.43325, "characteristics.samples_per_second.normalized_per_processor": 51.99, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "be3f36038e37499d", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 3677304, "90.00 percentile latency (ns)": 6286243, "90th percentile latency (ns)": 6286243, "95.00 percentile latency (ns)": 7081412, "97.00 percentile latency (ns)": 7586041, "99.00 percentile latency (ns)": 8651205, "99.90 percentile latency (ns)": 14696990, "Max latency (ns)": 23529535, "Mean latency (ns)": 4108056, "Min duration satisfied": "Yes", "Min latency (ns)": 2296187, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 242.74, "QPS w/o loadgen overhead": 243.42, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 47.81, "characteristics.good": 23905, "characteristics.power": 11.393116666666673, "characteristics.power.normalized_per_core": 0.09494263888888894, "characteristics.power.normalized_per_processor": 11.393116666666673, "characteristics.samples_per_second": 243.42, "characteristics.samples_per_second.normalized_per_core": 2.0284999999999997, "characteristics.samples_per_second.normalized_per_processor": 243.42, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6ac773f83648d0b2", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 9405957, "90.00 percentile latency (ns)": 17171012, "90th percentile latency (ns)": 17171012, "95.00 percentile latency (ns)": 18925224, "97.00 percentile latency (ns)": 19824850, "99.00 percentile latency (ns)": 21274900, "99.90 percentile latency (ns)": 27254976, "Max latency (ns)": 45878571, "Mean latency (ns)": 10388911, "Min duration satisfied": "Yes", "Min latency (ns)": 4879248, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 96.06, "QPS w/o loadgen overhead": 96.26, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.038, "characteristics.good": 32019, "characteristics.power": 11.300733333333339, "characteristics.power.normalized_per_core": 0.09417277777777783, "characteristics.power.normalized_per_processor": 11.300733333333339, "characteristics.samples_per_second": 96.26, "characteristics.samples_per_second.normalized_per_core": 0.8021666666666667, "characteristics.samples_per_second.normalized_per_processor": 96.26, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "62abe4342a7f47d6", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 5361392, "90.00 percentile latency (ns)": 9067761, "90th percentile latency (ns)": 9067761, "95.00 percentile latency (ns)": 10013196, "97.00 percentile latency (ns)": 10606584, "99.00 percentile latency (ns)": 11850194, "99.90 percentile latency (ns)": 17501932, "Max latency (ns)": 29377603, "Mean latency (ns)": 5865697, "Min duration satisfied": "Yes", "Min latency (ns)": 3334740, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 170.06, "QPS w/o loadgen overhead": 170.48, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 51.848, "characteristics.good": 25924, "characteristics.power": 11.525300000000003, "characteristics.power.normalized_per_core": 0.0960441666666667, "characteristics.power.normalized_per_processor": 11.525300000000003, "characteristics.samples_per_second": 170.48, "characteristics.samples_per_second.normalized_per_core": 1.4206666666666665, "characteristics.samples_per_second.normalized_per_processor": 170.48, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "fb347aeade213d69", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 6919237, "90.00 percentile latency (ns)": 15599944, "90th percentile latency (ns)": 15599944, "95.00 percentile latency (ns)": 17980800, "97.00 percentile latency (ns)": 19431934, "99.00 percentile latency (ns)": 21772809, "99.90 percentile latency (ns)": 25766161, "Max latency (ns)": 42719727, "Mean latency (ns)": 9497371, "Min duration satisfied": "Yes", "Min latency (ns)": 6069756, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 105.16, "QPS w/o loadgen overhead": 105.29, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.232, "characteristics.good": 29616, "characteristics.power": 12.356583333333342, "characteristics.power.normalized_per_core": 0.10297152777777785, "characteristics.power.normalized_per_processor": 12.356583333333342, "characteristics.samples_per_second": 105.29, "characteristics.samples_per_second.normalized_per_core": 0.8774166666666667, "characteristics.samples_per_second.normalized_per_processor": 105.29, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d73604724aeb057a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 16317443, "90.00 percentile latency (ns)": 19027152, "90th percentile latency (ns)": 19027152, "95.00 percentile latency (ns)": 20185059, "97.00 percentile latency (ns)": 21225831, "99.00 percentile latency (ns)": 23643261, "99.90 percentile latency (ns)": 29167434, "Max latency (ns)": 40206319, "Mean latency (ns)": 14827747, "Min duration satisfied": "Yes", "Min latency (ns)": 10391755, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 67.39, "QPS w/o loadgen overhead": 67.44, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.212, "characteristics.good": 34106, "characteristics.power": 15.187166666666656, "characteristics.power.normalized_per_core": 0.12655972222222214, "characteristics.power.normalized_per_processor": 15.187166666666656, "characteristics.samples_per_second": 67.44, "characteristics.samples_per_second.normalized_per_core": 0.5619999999999999, "characteristics.samples_per_second.normalized_per_processor": 67.44, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "a675ec4c33af553c", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 11612157, "90.00 percentile latency (ns)": 17841447, "90th percentile latency (ns)": 17841447, "95.00 percentile latency (ns)": 19515255, "97.00 percentile latency (ns)": 20497322, "99.00 percentile latency (ns)": 22325970, "99.90 percentile latency (ns)": 28337369, "Max latency (ns)": 40167122, "Mean latency (ns)": 12295824, "Min duration satisfied": "Yes", "Min latency (ns)": 6432357, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 81.2, "QPS w/o loadgen overhead": 81.33, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.072, "characteristics.good": 29536, "characteristics.power": 10.458983333333322, "characteristics.power.normalized_per_core": 0.08715819444444435, "characteristics.power.normalized_per_processor": 10.458983333333322, "characteristics.samples_per_second": 81.33, "characteristics.samples_per_second.normalized_per_core": 0.67775, "characteristics.samples_per_second.normalized_per_processor": 81.33, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "5c24e599471fc487", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 8684828, "90.00 percentile latency (ns)": 11332547, "90th percentile latency (ns)": 11332547, "95.00 percentile latency (ns)": 12259262, "97.00 percentile latency (ns)": 13002684, "99.00 percentile latency (ns)": 14996311, "99.90 percentile latency (ns)": 21437877, "Max latency (ns)": 29964834, "Mean latency (ns)": 8971845, "Min duration satisfied": "Yes", "Min latency (ns)": 5076206, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 111.32, "QPS w/o loadgen overhead": 111.46, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.386, "characteristics.good": 29193, "characteristics.power": 13.09439999999998, "characteristics.power.normalized_per_core": 0.10911999999999984, "characteristics.power.normalized_per_processor": 13.09439999999998, "characteristics.samples_per_second": 111.46, "characteristics.samples_per_second.normalized_per_core": 0.9288333333333333, "characteristics.samples_per_second.normalized_per_processor": 111.46, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "425319703e224062", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 25401560, "90.00 percentile latency (ns)": 28243154, "90th percentile latency (ns)": 28243154, "95.00 percentile latency (ns)": 30427206, "97.00 percentile latency (ns)": 32890097, "99.00 percentile latency (ns)": 36619795, "99.90 percentile latency (ns)": 42421804, "Max latency (ns)": 52279677, "Mean latency (ns)": 24299595, "Min duration satisfied": "Yes", "Min latency (ns)": 16664997, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 41.13, "QPS w/o loadgen overhead": 41.15, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.894, "characteristics.good": 36447, "characteristics.power": 14.408333333333337, "characteristics.power.normalized_per_core": 0.12006944444444448, "characteristics.power.normalized_per_processor": 14.408333333333337, "characteristics.samples_per_second": 41.15, "characteristics.samples_per_second.normalized_per_core": 0.34291666666666665, "characteristics.samples_per_second.normalized_per_processor": 41.15, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-large_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4d97a62053d99b8f", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 21638757, "90.00 percentile latency (ns)": 26620502, "90th percentile latency (ns)": 26620502, "95.00 percentile latency (ns)": 28375186, "97.00 percentile latency (ns)": 29948927, "99.00 percentile latency (ns)": 33310885, "99.90 percentile latency (ns)": 39238583, "Max latency (ns)": 46940520, "Mean latency (ns)": 20779044, "Min duration satisfied": "Yes", "Min latency (ns)": 14445751, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 48.09, "QPS w/o loadgen overhead": 48.13, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.598, "characteristics.good": 35299, "characteristics.power": 15.289366666666677, "characteristics.power.normalized_per_core": 0.12741138888888898, "characteristics.power.normalized_per_processor": 15.289366666666677, "characteristics.samples_per_second": 48.13, "characteristics.samples_per_second.normalized_per_core": 0.40108333333333335, "characteristics.samples_per_second.normalized_per_processor": 48.13, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 45.4545, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "84cd45ea9ce58f3b", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 12156208, "90.00 percentile latency (ns)": 19209358, "90th percentile latency (ns)": 19209358, "95.00 percentile latency (ns)": 21306625, "97.00 percentile latency (ns)": 22482206, "99.00 percentile latency (ns)": 24551728, "99.90 percentile latency (ns)": 29663169, "Max latency (ns)": 39305885, "Mean latency (ns)": 12840814, "Min duration satisfied": "Yes", "Min latency (ns)": 7168161, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 77.78, "QPS w/o loadgen overhead": 77.88, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.522, "characteristics.good": 28261, "characteristics.power": 11.042950000000012, "characteristics.power.normalized_per_core": 0.09202458333333344, "characteristics.power.normalized_per_processor": 11.042950000000012, "characteristics.samples_per_second": 77.88, "characteristics.samples_per_second.normalized_per_core": 0.6489999999999999, "characteristics.samples_per_second.normalized_per_processor": 77.88, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c714ed8bc5a67941", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 13183862, "90.00 percentile latency (ns)": 16820041, "90th percentile latency (ns)": 16820041, "95.00 percentile latency (ns)": 18348768, "97.00 percentile latency (ns)": 19682076, "99.00 percentile latency (ns)": 23300462, "99.90 percentile latency (ns)": 29537506, "Max latency (ns)": 39112014, "Mean latency (ns)": 13030530, "Min duration satisfied": "Yes", "Min latency (ns)": 9423102, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 76.67, "QPS w/o loadgen overhead": 76.74, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.726, "characteristics.good": 33863, "characteristics.power": 14.817950000000003, "characteristics.power.normalized_per_core": 0.12348291666666669, "characteristics.power.normalized_per_processor": 14.817950000000003, "characteristics.samples_per_second": 76.74, "characteristics.samples_per_second.normalized_per_core": 0.6395, "characteristics.samples_per_second.normalized_per_processor": 76.74, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "8a8dad378bde4542", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 3054078, "90.00 percentile latency (ns)": 5169718, "90th percentile latency (ns)": 5169718, "95.00 percentile latency (ns)": 5828690, "97.00 percentile latency (ns)": 6263461, "99.00 percentile latency (ns)": 7129670, "99.90 percentile latency (ns)": 13221158, "Max latency (ns)": 23021398, "Mean latency (ns)": 3453033, "Min duration satisfied": "Yes", "Min latency (ns)": 1877231, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 288.64, "QPS w/o loadgen overhead": 289.6, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 44.116, "characteristics.good": 22058, "characteristics.power": 11.036516666666676, "characteristics.power.normalized_per_core": 0.09197097222222231, "characteristics.power.normalized_per_processor": 11.036516666666676, "characteristics.samples_per_second": 289.6, "characteristics.samples_per_second.normalized_per_core": 2.4133333333333336, "characteristics.samples_per_second.normalized_per_processor": 289.6, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "433f01b884e56bdf", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 5426397, "90.00 percentile latency (ns)": 8963660, "90th percentile latency (ns)": 8963660, "95.00 percentile latency (ns)": 9650953, "97.00 percentile latency (ns)": 10018776, "99.00 percentile latency (ns)": 10827514, "99.90 percentile latency (ns)": 16961716, "Max latency (ns)": 34404668, "Mean latency (ns)": 5813186, "Min duration satisfied": "Yes", "Min latency (ns)": 2712975, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 171.59, "QPS w/o loadgen overhead": 172.02, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 45.444, "characteristics.good": 22722, "characteristics.power": 9.826733333333353, "characteristics.power.normalized_per_core": 0.08188944444444461, "characteristics.power.normalized_per_processor": 9.826733333333353, "characteristics.samples_per_second": 172.02, "characteristics.samples_per_second.normalized_per_core": 1.4335, "characteristics.samples_per_second.normalized_per_processor": 172.02, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "49169e086b402d92", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 13299268, "90.00 percentile latency (ns)": 16171810, "90th percentile latency (ns)": 16171810, "95.00 percentile latency (ns)": 17282483, "97.00 percentile latency (ns)": 18116479, "99.00 percentile latency (ns)": 20434741, "99.90 percentile latency (ns)": 25901646, "Max latency (ns)": 34435460, "Mean latency (ns)": 12911583, "Min duration satisfied": "Yes", "Min latency (ns)": 8315200, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 77.37, "QPS w/o loadgen overhead": 77.45, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.624, "characteristics.good": 35312, "characteristics.power": 17.540500000000016, "characteristics.power.normalized_per_core": 0.14617083333333347, "characteristics.power.normalized_per_processor": 17.540500000000016, "characteristics.samples_per_second": 77.45, "characteristics.samples_per_second.normalized_per_core": 0.6454166666666666, "characteristics.samples_per_second.normalized_per_processor": 77.45, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ca351522731811ba", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 15072774, "90.00 percentile latency (ns)": 21769317, "90th percentile latency (ns)": 21769317, "95.00 percentile latency (ns)": 23889386, "97.00 percentile latency (ns)": 25166720, "99.00 percentile latency (ns)": 27691411, "99.90 percentile latency (ns)": 32850660, "Max latency (ns)": 39276564, "Mean latency (ns)": 15421278, "Min duration satisfied": "Yes", "Min latency (ns)": 7661270, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 64.77, "QPS w/o loadgen overhead": 64.85, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.442, "characteristics.good": 31721, "characteristics.power": 10.916883333333331, "characteristics.power.normalized_per_core": 0.09097402777777776, "characteristics.power.normalized_per_processor": 10.916883333333331, "characteristics.samples_per_second": 64.85, "characteristics.samples_per_second.normalized_per_core": 0.5404166666666667, "characteristics.samples_per_second.normalized_per_processor": 64.85, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "92399bd7330c0151", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 14471323, "90.00 percentile latency (ns)": 22891244, "90th percentile latency (ns)": 22891244, "95.00 percentile latency (ns)": 24614615, "97.00 percentile latency (ns)": 25686247, "99.00 percentile latency (ns)": 28151314, "99.90 percentile latency (ns)": 32968805, "Max latency (ns)": 50251610, "Mean latency (ns)": 16120383, "Min duration satisfied": "Yes", "Min latency (ns)": 13768220, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 61.99, "QPS w/o loadgen overhead": 62.03, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.094, "characteristics.good": 35047, "characteristics.power": 16.701150000000005, "characteristics.power.normalized_per_core": 0.13917625000000006, "characteristics.power.normalized_per_processor": 16.701150000000005, "characteristics.samples_per_second": 62.03, "characteristics.samples_per_second.normalized_per_core": 0.5169166666666667, "characteristics.samples_per_second.normalized_per_processor": 62.03, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 58.8235, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "41c18e17d9fd154a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 65377951, "90.00 percentile latency (ns)": 80306648, "90th percentile latency (ns)": 80306648, "95.00 percentile latency (ns)": 87967292, "97.00 percentile latency (ns)": 96844419, "99.00 percentile latency (ns)": 105069029, "99.90 percentile latency (ns)": 119351638, "Max latency (ns)": 138295208, "Mean latency (ns)": 69557743, "Min duration satisfied": "Yes", "Min latency (ns)": 60440547, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 14.37, "QPS w/o loadgen overhead": 14.38, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.116, "characteristics.good": 37058, "characteristics.power": 19.467399999999998, "characteristics.power.normalized_per_core": 0.1622283333333333, "characteristics.power.normalized_per_processor": 19.467399999999998, "characteristics.samples_per_second": 14.38, "characteristics.samples_per_second.normalized_per_core": 0.11983333333333333, "characteristics.samples_per_second.normalized_per_processor": 14.38, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite2-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "22994f1ebb38a951", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 7066088, "90.00 percentile latency (ns)": 9579322, "90th percentile latency (ns)": 9579322, "95.00 percentile latency (ns)": 10509169, "97.00 percentile latency (ns)": 11256520, "99.00 percentile latency (ns)": 13017091, "99.90 percentile latency (ns)": 18873305, "Max latency (ns)": 35360521, "Mean latency (ns)": 7230024, "Min duration satisfied": "Yes", "Min latency (ns)": 4456575, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 138.12, "QPS w/o loadgen overhead": 138.31, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.432, "characteristics.good": 29716, "characteristics.power": 13.039033333333347, "characteristics.power.normalized_per_core": 0.10865861111111122, "characteristics.power.normalized_per_processor": 13.039033333333347, "characteristics.samples_per_second": 138.31, "characteristics.samples_per_second.normalized_per_core": 1.1525833333333333, "characteristics.samples_per_second.normalized_per_processor": 138.31, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ac28f988e26f469a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 9014717, "90.00 percentile latency (ns)": 11143895, "90th percentile latency (ns)": 11143895, "95.00 percentile latency (ns)": 12035229, "97.00 percentile latency (ns)": 12752251, "99.00 percentile latency (ns)": 14712781, "99.90 percentile latency (ns)": 21330438, "Max latency (ns)": 33308893, "Mean latency (ns)": 9005844, "Min duration satisfied": "Yes", "Min latency (ns)": 5451814, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 110.92, "QPS w/o loadgen overhead": 111.04, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.458, "characteristics.good": 30729, "characteristics.power": 13.548083333333349, "characteristics.power.normalized_per_core": 0.11290069444444457, "characteristics.power.normalized_per_processor": 13.548083333333349, "characteristics.samples_per_second": 111.04, "characteristics.samples_per_second.normalized_per_core": 0.9253333333333333, "characteristics.samples_per_second.normalized_per_processor": 111.04, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "1b6d46bc815ea585", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 15084221, "90.00 percentile latency (ns)": 22346717, "90th percentile latency (ns)": 22346717, "95.00 percentile latency (ns)": 24503903, "97.00 percentile latency (ns)": 25837850, "99.00 percentile latency (ns)": 28158624, "99.90 percentile latency (ns)": 33398184, "Max latency (ns)": 44724351, "Mean latency (ns)": 15498800, "Min duration satisfied": "Yes", "Min latency (ns)": 8422581, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 64.44, "QPS w/o loadgen overhead": 64.52, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.544, "characteristics.good": 29272, "characteristics.power": 11.181216666666657, "characteristics.power.normalized_per_core": 0.09317680555555548, "characteristics.power.normalized_per_processor": 11.181216666666657, "characteristics.samples_per_second": 64.52, "characteristics.samples_per_second.normalized_per_core": 0.5376666666666666, "characteristics.samples_per_second.normalized_per_processor": 64.52, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "53445fae8201c1cd", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 10269855, "90.00 percentile latency (ns)": 18819545, "90th percentile latency (ns)": 18819545, "95.00 percentile latency (ns)": 20717139, "97.00 percentile latency (ns)": 21713854, "99.00 percentile latency (ns)": 23387696, "99.90 percentile latency (ns)": 28933917, "Max latency (ns)": 41476715, "Mean latency (ns)": 11256663, "Min duration satisfied": "Yes", "Min latency (ns)": 5190813, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 88.67, "QPS w/o loadgen overhead": 88.84, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.036, "characteristics.good": 33018, "characteristics.power": 11.58044999999999, "characteristics.power.normalized_per_core": 0.09650374999999992, "characteristics.power.normalized_per_processor": 11.58044999999999, "characteristics.samples_per_second": 88.84, "characteristics.samples_per_second.normalized_per_core": 0.7403333333333334, "characteristics.samples_per_second.normalized_per_processor": 88.84, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "557649e78da10e11", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 23427399, "90.00 percentile latency (ns)": 26739832, "90th percentile latency (ns)": 26739832, "95.00 percentile latency (ns)": 28830304, "97.00 percentile latency (ns)": 31265283, "99.00 percentile latency (ns)": 34962493, "99.90 percentile latency (ns)": 40453465, "Max latency (ns)": 46891204, "Mean latency (ns)": 22150857, "Min duration satisfied": "Yes", "Min latency (ns)": 14406313, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 45.11, "QPS w/o loadgen overhead": 45.14, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.642, "characteristics.good": 38321, "characteristics.power": 17.49761666666666, "characteristics.power.normalized_per_core": 0.14581347222222216, "characteristics.power.normalized_per_processor": 17.49761666666666, "characteristics.samples_per_second": 45.14, "characteristics.samples_per_second.normalized_per_core": 0.37616666666666665, "characteristics.samples_per_second.normalized_per_processor": 45.14, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite1-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d6885538d1704854", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 18691505, "90.00 percentile latency (ns)": 21513713, "90th percentile latency (ns)": 21513713, "95.00 percentile latency (ns)": 23185751, "97.00 percentile latency (ns)": 24707564, "99.00 percentile latency (ns)": 28369645, "99.90 percentile latency (ns)": 35173316, "Max latency (ns)": 43619398, "Mean latency (ns)": 18017033, "Min duration satisfied": "Yes", "Min latency (ns)": 12287231, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 55.46, "QPS w/o loadgen overhead": 55.5, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.678, "characteristics.good": 34839, "characteristics.power": 14.5458, "characteristics.power.normalized_per_core": 0.121215, "characteristics.power.normalized_per_processor": 14.5458, "characteristics.samples_per_second": 55.5, "characteristics.samples_per_second.normalized_per_core": 0.4625, "characteristics.samples_per_second.normalized_per_processor": 55.5, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "aa4923bcf1077d81", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 11188799, "90.00 percentile latency (ns)": 13809618, "90th percentile latency (ns)": 13809618, "95.00 percentile latency (ns)": 14956993, "97.00 percentile latency (ns)": 15801505, "99.00 percentile latency (ns)": 18243535, "99.90 percentile latency (ns)": 24507805, "Max latency (ns)": 40071028, "Mean latency (ns)": 11141577, "Min duration satisfied": "Yes", "Min latency (ns)": 6739135, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 89.64, "QPS w/o loadgen overhead": 89.75, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.262, "characteristics.good": 34631, "characteristics.power": 16.58279999999998, "characteristics.power.normalized_per_core": 0.13818999999999984, "characteristics.power.normalized_per_processor": 16.58279999999998, "characteristics.samples_per_second": 89.75, "characteristics.samples_per_second.normalized_per_core": 0.7479166666666667, "characteristics.samples_per_second.normalized_per_processor": 89.75, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "e3414cb68ead429f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 18166482, "90.00 percentile latency (ns)": 22564149, "90th percentile latency (ns)": 22564149, "95.00 percentile latency (ns)": 24078395, "97.00 percentile latency (ns)": 25632547, "99.00 percentile latency (ns)": 29076928, "99.90 percentile latency (ns)": 35207230, "Max latency (ns)": 44361745, "Mean latency (ns)": 17517179, "Min duration satisfied": "Yes", "Min latency (ns)": 12362604, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 57.04, "QPS w/o loadgen overhead": 57.09, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.76, "characteristics.good": 34380, "characteristics.power": 15.323083333333324, "characteristics.power.normalized_per_core": 0.12769236111111104, "characteristics.power.normalized_per_processor": 15.323083333333324, "characteristics.samples_per_second": 57.09, "characteristics.samples_per_second.normalized_per_core": 0.47575, "characteristics.samples_per_second.normalized_per_processor": 57.09, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c8b61398e6f32fe1", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 6707135, "90.00 percentile latency (ns)": 7869873, "90th percentile latency (ns)": 7869873, "95.00 percentile latency (ns)": 8294242, "97.00 percentile latency (ns)": 8671345, "99.00 percentile latency (ns)": 10425309, "99.90 percentile latency (ns)": 18044612, "Max latency (ns)": 25348957, "Mean latency (ns)": 6666517, "Min duration satisfied": "Yes", "Min latency (ns)": 4439390, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 149.7, "QPS w/o loadgen overhead": 150.0, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.33, "characteristics.good": 31665, "characteristics.power": 14.165716666666652, "characteristics.power.normalized_per_core": 0.11804763888888876, "characteristics.power.normalized_per_processor": 14.165716666666652, "characteristics.samples_per_second": 150.0, "characteristics.samples_per_second.normalized_per_core": 1.25, "characteristics.samples_per_second.normalized_per_processor": 150.0, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "050c04010b55ae08", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 4374033, "90.00 percentile latency (ns)": 8984164, "90th percentile latency (ns)": 8984164, "95.00 percentile latency (ns)": 9932393, "97.00 percentile latency (ns)": 10452030, "99.00 percentile latency (ns)": 11288008, "99.90 percentile latency (ns)": 16996958, "Max latency (ns)": 26221169, "Mean latency (ns)": 5105663, "Min duration satisfied": "Yes", "Min latency (ns)": 2476708, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 195.21, "QPS w/o loadgen overhead": 195.86, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 57.918, "characteristics.good": 28959, "characteristics.power": 11.293549999999998, "characteristics.power.normalized_per_core": 0.09411291666666664, "characteristics.power.normalized_per_processor": 11.293549999999998, "characteristics.samples_per_second": 195.86, "characteristics.samples_per_second.normalized_per_core": 1.6321666666666668, "characteristics.samples_per_second.normalized_per_processor": 195.86, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "a9a311da22bfed80", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 14735294, "90.00 percentile latency (ns)": 19582731, "90th percentile latency (ns)": 19582731, "95.00 percentile latency (ns)": 20845665, "97.00 percentile latency (ns)": 21829479, "99.00 percentile latency (ns)": 24157958, "99.90 percentile latency (ns)": 29629823, "Max latency (ns)": 42818491, "Mean latency (ns)": 14747264, "Min duration satisfied": "Yes", "Min latency (ns)": 10424643, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 67.74, "QPS w/o loadgen overhead": 67.81, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.674, "characteristics.good": 35837, "characteristics.power": 18.93398333333333, "characteristics.power.normalized_per_core": 0.1577831944444444, "characteristics.power.normalized_per_processor": 18.93398333333333, "characteristics.samples_per_second": 67.81, "characteristics.samples_per_second.normalized_per_core": 0.5650833333333334, "characteristics.samples_per_second.normalized_per_processor": 67.81, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ba83042c4baf8a09", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 7167125, "90.00 percentile latency (ns)": 11471660, "90th percentile latency (ns)": 11471660, "95.00 percentile latency (ns)": 12801190, "97.00 percentile latency (ns)": 13530981, "99.00 percentile latency (ns)": 14768806, "99.90 percentile latency (ns)": 21209858, "Max latency (ns)": 33246067, "Mean latency (ns)": 7829000, "Min duration satisfied": "Yes", "Min latency (ns)": 4356056, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 127.48, "QPS w/o loadgen overhead": 127.73, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 43.454, "characteristics.good": 21727, "characteristics.power": 10.345983333333335, "characteristics.power.normalized_per_core": 0.08621652777777779, "characteristics.power.normalized_per_processor": 10.345983333333335, "characteristics.samples_per_second": 127.73, "characteristics.samples_per_second.normalized_per_core": 1.0644166666666668, "characteristics.samples_per_second.normalized_per_processor": 127.73, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "5625245af8c7ff68", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 42843911, "90.00 percentile latency (ns)": 77314084, "90th percentile latency (ns)": 77314084, "95.00 percentile latency (ns)": 81061939, "97.00 percentile latency (ns)": 83912242, "99.00 percentile latency (ns)": 91711307, "99.90 percentile latency (ns)": 99747516, "Max latency (ns)": 105543697, "Mean latency (ns)": 50065569, "Min duration satisfied": "Yes", "Min latency (ns)": 40976028, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 19.97, "QPS w/o loadgen overhead": 19.97, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 80.676, "characteristics.good": 40338, "characteristics.power": 22.068283333333323, "characteristics.power.normalized_per_core": 0.18390236111111102, "characteristics.power.normalized_per_processor": 22.068283333333323, "characteristics.samples_per_second": 19.97, "characteristics.samples_per_second.normalized_per_core": 0.16641666666666666, "characteristics.samples_per_second.normalized_per_processor": 19.97, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite4-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "048aba2586c99832", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 20279562, "90.00 percentile latency (ns)": 22373190, "90th percentile latency (ns)": 22373190, "95.00 percentile latency (ns)": 23832088, "97.00 percentile latency (ns)": 25654868, "99.00 percentile latency (ns)": 29569717, "99.90 percentile latency (ns)": 34922193, "Max latency (ns)": 42362597, "Mean latency (ns)": 19235794, "Min duration satisfied": "Yes", "Min latency (ns)": 12823409, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 51.95, "QPS w/o loadgen overhead": 51.99, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.33, "characteristics.good": 35165, "characteristics.power": 14.515133333333337, "characteristics.power.normalized_per_core": 0.12095944444444447, "characteristics.power.normalized_per_processor": 14.515133333333337, "characteristics.samples_per_second": 51.99, "characteristics.samples_per_second.normalized_per_core": 0.43325, "characteristics.samples_per_second.normalized_per_processor": 51.99, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "3a93836c042fe533", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 15980051, "90.00 percentile latency (ns)": 18622335, "90th percentile latency (ns)": 18622335, "95.00 percentile latency (ns)": 20189382, "97.00 percentile latency (ns)": 21596163, "99.00 percentile latency (ns)": 25275196, "99.90 percentile latency (ns)": 31415434, "Max latency (ns)": 37507761, "Mean latency (ns)": 15500332, "Min duration satisfied": "Yes", "Min latency (ns)": 10414448, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 64.46, "QPS w/o loadgen overhead": 64.51, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.4, "characteristics.good": 33700, "characteristics.power": 14.452749999999996, "characteristics.power.normalized_per_core": 0.12043958333333331, "characteristics.power.normalized_per_processor": 14.452749999999996, "characteristics.samples_per_second": 64.51, "characteristics.samples_per_second.normalized_per_core": 0.5375833333333334, "characteristics.samples_per_second.normalized_per_processor": 64.51, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 58.8235, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "30e621a4663a5fc5", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 14089794, "90.00 percentile latency (ns)": 16884121, "90th percentile latency (ns)": 16884121, "95.00 percentile latency (ns)": 17998030, "97.00 percentile latency (ns)": 19002975, "99.00 percentile latency (ns)": 21325791, "99.90 percentile latency (ns)": 26789993, "Max latency (ns)": 36270027, "Mean latency (ns)": 13346982, "Min duration satisfied": "Yes", "Min latency (ns)": 9114510, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 74.86, "QPS w/o loadgen overhead": 74.92, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.208, "characteristics.good": 33604, "characteristics.power": 14.951550000000001, "characteristics.power.normalized_per_core": 0.12459625, "characteristics.power.normalized_per_processor": 14.951550000000001, "characteristics.samples_per_second": 74.92, "characteristics.samples_per_second.normalized_per_core": 0.6243333333333333, "characteristics.samples_per_second.normalized_per_processor": 74.92, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "dfd8473bbf59e863", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 8567772, "90.00 percentile latency (ns)": 12464258, "90th percentile latency (ns)": 12464258, "95.00 percentile latency (ns)": 13784093, "97.00 percentile latency (ns)": 14766151, "99.00 percentile latency (ns)": 16856141, "99.90 percentile latency (ns)": 21979679, "Max latency (ns)": 33897174, "Mean latency (ns)": 8673338, "Min duration satisfied": "Yes", "Min latency (ns)": 5023483, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 115.08, "QPS w/o loadgen overhead": 115.3, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.348, "characteristics.good": 32174, "characteristics.power": 13.689783333333335, "characteristics.power.normalized_per_core": 0.11408152777777779, "characteristics.power.normalized_per_processor": 13.689783333333335, "characteristics.samples_per_second": 115.3, "characteristics.samples_per_second.normalized_per_core": 0.9608333333333333, "characteristics.samples_per_second.normalized_per_processor": 115.3, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "96b8214b199cfaa2", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 6953929, "90.00 percentile latency (ns)": 11605453, "90th percentile latency (ns)": 11605453, "95.00 percentile latency (ns)": 12916390, "97.00 percentile latency (ns)": 13734923, "99.00 percentile latency (ns)": 15336684, "99.90 percentile latency (ns)": 20151576, "Max latency (ns)": 29972833, "Mean latency (ns)": 7291743, "Min duration satisfied": "Yes", "Min latency (ns)": 3930822, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 136.82, "QPS w/o loadgen overhead": 137.14, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.246, "characteristics.good": 31623, "characteristics.power": 12.52235, "characteristics.power.normalized_per_core": 0.10435291666666666, "characteristics.power.normalized_per_processor": 12.52235, "characteristics.samples_per_second": 137.14, "characteristics.samples_per_second.normalized_per_core": 1.1428333333333331, "characteristics.samples_per_second.normalized_per_processor": 137.14, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "8d889a42045d9bf2", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 6728828, "90.00 percentile latency (ns)": 15480466, "90th percentile latency (ns)": 15480466, "95.00 percentile latency (ns)": 17199096, "97.00 percentile latency (ns)": 18036420, "99.00 percentile latency (ns)": 19414229, "99.90 percentile latency (ns)": 24941675, "Max latency (ns)": 35664764, "Mean latency (ns)": 8675916, "Min duration satisfied": "Yes", "Min latency (ns)": 4697863, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 115.0, "QPS w/o loadgen overhead": 115.26, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.668, "characteristics.good": 30834, "characteristics.power": 11.596083333333338, "characteristics.power.normalized_per_core": 0.09663402777777781, "characteristics.power.normalized_per_processor": 11.596083333333338, "characteristics.samples_per_second": 115.26, "characteristics.samples_per_second.normalized_per_core": 0.9605, "characteristics.samples_per_second.normalized_per_processor": 115.26, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "74cce9cd4db4a686", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 6655356, "90.00 percentile latency (ns)": 9213251, "90th percentile latency (ns)": 9213251, "95.00 percentile latency (ns)": 10258059, "97.00 percentile latency (ns)": 11034603, "99.00 percentile latency (ns)": 13039935, "99.90 percentile latency (ns)": 19488191, "Max latency (ns)": 32487264, "Mean latency (ns)": 6898559, "Min duration satisfied": "Yes", "Min latency (ns)": 4174210, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 144.67, "QPS w/o loadgen overhead": 144.96, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.384, "characteristics.good": 33192, "characteristics.power": 15.677533333333349, "characteristics.power.normalized_per_core": 0.13064611111111124, "characteristics.power.normalized_per_processor": 15.677533333333349, "characteristics.samples_per_second": 144.96, "characteristics.samples_per_second.normalized_per_core": 1.208, "characteristics.samples_per_second.normalized_per_processor": 144.96, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ae4cc290d99a9664", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 8312427, "90.00 percentile latency (ns)": 13158214, "90th percentile latency (ns)": 13158214, "95.00 percentile latency (ns)": 14423214, "97.00 percentile latency (ns)": 15080963, "99.00 percentile latency (ns)": 16391725, "99.90 percentile latency (ns)": 23061924, "Max latency (ns)": 32437744, "Mean latency (ns)": 8934767, "Min duration satisfied": "Yes", "Min latency (ns)": 4638805, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 111.7, "QPS w/o loadgen overhead": 111.92, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.296, "characteristics.good": 24648, "characteristics.power": 10.017416666666668, "characteristics.power.normalized_per_core": 0.08347847222222222, "characteristics.power.normalized_per_processor": 10.017416666666668, "characteristics.samples_per_second": 111.92, "characteristics.samples_per_second.normalized_per_core": 0.9326666666666666, "characteristics.samples_per_second.normalized_per_processor": 111.92, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "60b4804b77272e20", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 13132532, "90.00 percentile latency (ns)": 19934148, "90th percentile latency (ns)": 19934148, "95.00 percentile latency (ns)": 21830851, "97.00 percentile latency (ns)": 23062500, "99.00 percentile latency (ns)": 25190971, "99.90 percentile latency (ns)": 30704542, "Max latency (ns)": 37327535, "Mean latency (ns)": 13500146, "Min duration satisfied": "Yes", "Min latency (ns)": 6714481, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 73.97, "QPS w/o loadgen overhead": 74.07, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 62.082, "characteristics.good": 31041, "characteristics.power": 10.82018333333334, "characteristics.power.normalized_per_core": 0.0901681944444445, "characteristics.power.normalized_per_processor": 10.82018333333334, "characteristics.samples_per_second": 74.07, "characteristics.samples_per_second.normalized_per_core": 0.61725, "characteristics.samples_per_second.normalized_per_processor": 74.07, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6b44544708e2429d", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 8981577, "90.00 percentile latency (ns)": 16223351, "90th percentile latency (ns)": 16223351, "95.00 percentile latency (ns)": 17168372, "97.00 percentile latency (ns)": 17699442, "99.00 percentile latency (ns)": 18855654, "99.90 percentile latency (ns)": 26407425, "Max latency (ns)": 40597409, "Mean latency (ns)": 9762244, "Min duration satisfied": "Yes", "Min latency (ns)": 4147517, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 102.19, "QPS w/o loadgen overhead": 102.44, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.224, "characteristics.good": 30612, "characteristics.power": 10.478566666666671, "characteristics.power.normalized_per_core": 0.08732138888888892, "characteristics.power.normalized_per_processor": 10.478566666666671, "characteristics.samples_per_second": 102.44, "characteristics.samples_per_second.normalized_per_core": 0.8536666666666667, "characteristics.samples_per_second.normalized_per_processor": 102.44, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c92b5b42c6ca80d3", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 4952592, "90.00 percentile latency (ns)": 7597833, "90th percentile latency (ns)": 7597833, "95.00 percentile latency (ns)": 8061587, "97.00 percentile latency (ns)": 8354079, "99.00 percentile latency (ns)": 9160097, "99.90 percentile latency (ns)": 15543853, "Max latency (ns)": 26729794, "Mean latency (ns)": 5155334, "Min duration satisfied": "Yes", "Min latency (ns)": 2192828, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 193.37, "QPS w/o loadgen overhead": 193.97, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 42.348, "characteristics.good": 21174, "characteristics.power": 9.182050000000002, "characteristics.power.normalized_per_core": 0.07651708333333335, "characteristics.power.normalized_per_processor": 9.182050000000002, "characteristics.samples_per_second": 193.97, "characteristics.samples_per_second.normalized_per_core": 1.6164166666666666, "characteristics.samples_per_second.normalized_per_processor": 193.97, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "343fe4392b39b851", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 19719265, "90.00 percentile latency (ns)": 24209468, "90th percentile latency (ns)": 24209468, "95.00 percentile latency (ns)": 26163714, "97.00 percentile latency (ns)": 27624309, "99.00 percentile latency (ns)": 30940732, "99.90 percentile latency (ns)": 36865110, "Max latency (ns)": 45531215, "Mean latency (ns)": 19093914, "Min duration satisfied": "Yes", "Min latency (ns)": 11692574, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 52.33, "QPS w/o loadgen overhead": 52.37, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.012, "characteristics.good": 37506, "characteristics.power": 17.39811666666665, "characteristics.power.normalized_per_core": 0.14498430555555541, "characteristics.power.normalized_per_processor": 17.39811666666665, "characteristics.samples_per_second": 52.37, "characteristics.samples_per_second.normalized_per_core": 0.4364166666666666, "characteristics.samples_per_second.normalized_per_processor": 52.37, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.3-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.3-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.3_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 47.619, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "000a968a44ffa80f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 11759745, "90.00 percentile latency (ns)": 13779228, "90th percentile latency (ns)": 13779228, "95.00 percentile latency (ns)": 14434877, "97.00 percentile latency (ns)": 15812562, "99.00 percentile latency (ns)": 19428533, "99.90 percentile latency (ns)": 26021404, "Max latency (ns)": 42721549, "Mean latency (ns)": 11554274, "Min duration satisfied": "Yes", "Min latency (ns)": 7930779, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 86.45, "QPS w/o loadgen overhead": 86.55, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.062, "characteristics.good": 32031, "characteristics.power": 13.888533333333339, "characteristics.power.normalized_per_core": 0.11573777777777781, "characteristics.power.normalized_per_processor": 13.888533333333339, "characteristics.samples_per_second": 86.55, "characteristics.samples_per_second.normalized_per_core": 0.72125, "characteristics.samples_per_second.normalized_per_processor": 86.55, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-small_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "f4b9733b68fa68dc", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 30682885, "90.00 percentile latency (ns)": 34792199, "90th percentile latency (ns)": 34792199, "95.00 percentile latency (ns)": 37222849, "97.00 percentile latency (ns)": 39734486, "99.00 percentile latency (ns)": 43286468, "99.90 percentile latency (ns)": 48818565, "Max latency (ns)": 65067644, "Mean latency (ns)": 28829886, "Min duration satisfied": "Yes", "Min latency (ns)": 18089721, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 34.67, "QPS w/o loadgen overhead": 34.69, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 77.542, "characteristics.good": 38771, "characteristics.power": 17.892549999999996, "characteristics.power.normalized_per_core": 0.1491045833333333, "characteristics.power.normalized_per_processor": 17.892549999999996, "characteristics.samples_per_second": 34.69, "characteristics.samples_per_second.normalized_per_core": 0.2890833333333333, "characteristics.samples_per_second.normalized_per_processor": 34.69, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite2-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ef4c73ebfca34552", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 6010461, "90.00 percentile latency (ns)": 10756071, "90th percentile latency (ns)": 10756071, "95.00 percentile latency (ns)": 11920467, "97.00 percentile latency (ns)": 12613521, "99.00 percentile latency (ns)": 13718906, "99.90 percentile latency (ns)": 18756269, "Max latency (ns)": 35290755, "Mean latency (ns)": 6516395, "Min duration satisfied": "Yes", "Min latency (ns)": 3244872, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 153.15, "QPS w/o loadgen overhead": 153.46, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.738, "characteristics.good": 28369, "characteristics.power": 10.825700000000007, "characteristics.power.normalized_per_core": 0.09021416666666672, "characteristics.power.normalized_per_processor": 10.825700000000007, "characteristics.samples_per_second": 153.46, "characteristics.samples_per_second.normalized_per_core": 1.2788333333333335, "characteristics.samples_per_second.normalized_per_processor": 153.46, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "0d68d34d7eac7679", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 10692370, "90.00 percentile latency (ns)": 16512491, "90th percentile latency (ns)": 16512491, "95.00 percentile latency (ns)": 17999407, "97.00 percentile latency (ns)": 18819831, "99.00 percentile latency (ns)": 20293247, "99.90 percentile latency (ns)": 26068104, "Max latency (ns)": 37088692, "Mean latency (ns)": 11261197, "Min duration satisfied": "Yes", "Min latency (ns)": 5632484, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 88.65, "QPS w/o loadgen overhead": 88.8, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 57.054, "characteristics.good": 28527, "characteristics.power": 10.17764999999999, "characteristics.power.normalized_per_core": 0.08481374999999991, "characteristics.power.normalized_per_processor": 10.17764999999999, "characteristics.samples_per_second": 88.8, "characteristics.samples_per_second.normalized_per_core": 0.74, "characteristics.samples_per_second.normalized_per_processor": 88.8, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "45cec69576e408bb", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 19989460, "90.00 percentile latency (ns)": 24490225, "90th percentile latency (ns)": 24490225, "95.00 percentile latency (ns)": 26612464, "97.00 percentile latency (ns)": 28232924, "99.00 percentile latency (ns)": 31409294, "99.90 percentile latency (ns)": 37952708, "Max latency (ns)": 45077012, "Mean latency (ns)": 18439733, "Min duration satisfied": "Yes", "Min latency (ns)": 12318420, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 54.19, "QPS w/o loadgen overhead": 54.23, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.69, "characteristics.good": 37845, "characteristics.power": 18.247466666666668, "characteristics.power.normalized_per_core": 0.15206222222222224, "characteristics.power.normalized_per_processor": 18.247466666666668, "characteristics.samples_per_second": 54.23, "characteristics.samples_per_second.normalized_per_core": 0.45191666666666663, "characteristics.samples_per_second.normalized_per_processor": 54.23, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.4-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.4-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.4_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 47.619, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "f3b45578c4fe083c", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 8218933, "90.00 percentile latency (ns)": 12627343, "90th percentile latency (ns)": 12627343, "95.00 percentile latency (ns)": 13270231, "97.00 percentile latency (ns)": 13673185, "99.00 percentile latency (ns)": 14723282, "99.90 percentile latency (ns)": 22343941, "Max latency (ns)": 31051694, "Mean latency (ns)": 8366573, "Min duration satisfied": "Yes", "Min latency (ns)": 3510165, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 119.13, "QPS w/o loadgen overhead": 119.52, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 52.446, "characteristics.good": 26223, "characteristics.power": 9.484166666666672, "characteristics.power.normalized_per_core": 0.07903472222222227, "characteristics.power.normalized_per_processor": 9.484166666666672, "characteristics.samples_per_second": 119.52, "characteristics.samples_per_second.normalized_per_core": 0.996, "characteristics.samples_per_second.normalized_per_processor": 119.52, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "8eb33372c64366b5", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 15532437, "90.00 percentile latency (ns)": 20635328, "90th percentile latency (ns)": 20635328, "95.00 percentile latency (ns)": 22184393, "97.00 percentile latency (ns)": 23955689, "99.00 percentile latency (ns)": 28053523, "99.90 percentile latency (ns)": 34332520, "Max latency (ns)": 51188034, "Mean latency (ns)": 16106030, "Min duration satisfied": "Yes", "Min latency (ns)": 11347498, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 62.03, "QPS w/o loadgen overhead": 62.09, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.724, "characteristics.good": 36862, "characteristics.power": 16.012400000000003, "characteristics.power.normalized_per_core": 0.1334366666666667, "characteristics.power.normalized_per_processor": 16.012400000000003, "characteristics.samples_per_second": 62.09, "characteristics.samples_per_second.normalized_per_core": 0.5174166666666667, "characteristics.samples_per_second.normalized_per_processor": 62.09, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-large_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "f75d4f5884b0493e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 7745481, "90.00 percentile latency (ns)": 9936454, "90th percentile latency (ns)": 9936454, "95.00 percentile latency (ns)": 10880278, "97.00 percentile latency (ns)": 11622670, "99.00 percentile latency (ns)": 13646061, "99.90 percentile latency (ns)": 20215803, "Max latency (ns)": 40741673, "Mean latency (ns)": 7777639, "Min duration satisfied": "Yes", "Min latency (ns)": 4640168, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 128.35, "QPS w/o loadgen overhead": 128.57, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.604, "characteristics.good": 33302, "characteristics.power": 16.382266666666677, "characteristics.power.normalized_per_core": 0.13651888888888897, "characteristics.power.normalized_per_processor": 16.382266666666677, "characteristics.samples_per_second": 128.57, "characteristics.samples_per_second.normalized_per_core": 1.0714166666666667, "characteristics.samples_per_second.normalized_per_processor": 128.57, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4a65a8f5391a06fc", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 10104410, "90.00 percentile latency (ns)": 12534049, "90th percentile latency (ns)": 12534049, "95.00 percentile latency (ns)": 13650831, "97.00 percentile latency (ns)": 14475440, "99.00 percentile latency (ns)": 16692717, "99.90 percentile latency (ns)": 22921833, "Max latency (ns)": 31015823, "Mean latency (ns)": 9883727, "Min duration satisfied": "Yes", "Min latency (ns)": 5907089, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 101.04, "QPS w/o loadgen overhead": 101.18, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.106, "characteristics.good": 34553, "characteristics.power": 17.011366666666657, "characteristics.power.normalized_per_core": 0.1417613888888888, "characteristics.power.normalized_per_processor": 17.011366666666657, "characteristics.samples_per_second": 101.18, "characteristics.samples_per_second.normalized_per_core": 0.8431666666666667, "characteristics.samples_per_second.normalized_per_processor": 101.18, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "7d7975e0b347c1ad", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 9437371, "90.00 percentile latency (ns)": 15100576, "90th percentile latency (ns)": 15100576, "95.00 percentile latency (ns)": 15907332, "97.00 percentile latency (ns)": 16363506, "99.00 percentile latency (ns)": 17497635, "99.90 percentile latency (ns)": 24778347, "Max latency (ns)": 34367467, "Mean latency (ns)": 9722363, "Min duration satisfied": "Yes", "Min latency (ns)": 3953152, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 102.59, "QPS w/o loadgen overhead": 102.86, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.822, "characteristics.good": 29911, "characteristics.power": 9.916283333333334, "characteristics.power.normalized_per_core": 0.08263569444444445, "characteristics.power.normalized_per_processor": 9.916283333333334, "characteristics.samples_per_second": 102.86, "characteristics.samples_per_second.normalized_per_core": 0.8571666666666666, "characteristics.samples_per_second.normalized_per_processor": 102.86, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "504bf43f4d4ea884", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 12898995, "90.00 percentile latency (ns)": 15371293, "90th percentile latency (ns)": 15371293, "95.00 percentile latency (ns)": 16330116, "97.00 percentile latency (ns)": 17345965, "99.00 percentile latency (ns)": 21205513, "99.90 percentile latency (ns)": 27676565, "Max latency (ns)": 43520671, "Mean latency (ns)": 12414804, "Min duration satisfied": "Yes", "Min latency (ns)": 8768201, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 80.45, "QPS w/o loadgen overhead": 80.55, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.336, "characteristics.good": 34168, "characteristics.power": 14.919100000000016, "characteristics.power.normalized_per_core": 0.12432583333333347, "characteristics.power.normalized_per_processor": 14.919100000000016, "characteristics.samples_per_second": 80.55, "characteristics.samples_per_second.normalized_per_core": 0.67125, "characteristics.samples_per_second.normalized_per_processor": 80.55, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-small_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "1579cf1d404dbb3a", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 11994436, "90.00 percentile latency (ns)": 18886897, "90th percentile latency (ns)": 18886897, "95.00 percentile latency (ns)": 21042154, "97.00 percentile latency (ns)": 22508853, "99.00 percentile latency (ns)": 25641520, "99.90 percentile latency (ns)": 30864784, "Max latency (ns)": 41448959, "Mean latency (ns)": 12456331, "Min duration satisfied": "Yes", "Min latency (ns)": 6964140, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 80.16, "QPS w/o loadgen overhead": 80.28, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.834, "characteristics.good": 34417, "characteristics.power": 13.676800000000005, "characteristics.power.normalized_per_core": 0.11397333333333338, "characteristics.power.normalized_per_processor": 13.676800000000005, "characteristics.samples_per_second": 80.28, "characteristics.samples_per_second.normalized_per_core": 0.669, "characteristics.samples_per_second.normalized_per_processor": 80.28, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "e41b6ac0029da61e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 29522296, "90.00 percentile latency (ns)": 39275905, "90th percentile latency (ns)": 39275905, "95.00 percentile latency (ns)": 42337042, "97.00 percentile latency (ns)": 45395133, "99.00 percentile latency (ns)": 53001585, "99.90 percentile latency (ns)": 64958075, "Max latency (ns)": 77029680, "Mean latency (ns)": 32111771, "Min duration satisfied": "Yes", "Min latency (ns)": 27315107, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 31.13, "QPS w/o loadgen overhead": 31.14, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.254, "characteristics.good": 35627, "characteristics.power": 19.90409999999999, "characteristics.power.normalized_per_core": 0.16586749999999992, "characteristics.power.normalized_per_processor": 19.90409999999999, "characteristics.samples_per_second": 31.14, "characteristics.samples_per_second.normalized_per_core": 0.2595, "characteristics.samples_per_second.normalized_per_processor": 31.14, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite0-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "29ae8621d525ef76", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 27083021, "90.00 percentile latency (ns)": 49307993, "90th percentile latency (ns)": 49307993, "95.00 percentile latency (ns)": 51988858, "97.00 percentile latency (ns)": 54509321, "99.00 percentile latency (ns)": 59493648, "99.90 percentile latency (ns)": 66443721, "Max latency (ns)": 84040221, "Mean latency (ns)": 34796353, "Min duration satisfied": "Yes", "Min latency (ns)": 25253254, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 28.72, "QPS w/o loadgen overhead": 28.74, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 79.268, "characteristics.good": 39634, "characteristics.power": 19.979716666666665, "characteristics.power.normalized_per_core": 0.16649763888888888, "characteristics.power.normalized_per_processor": 19.979716666666665, "characteristics.samples_per_second": 28.74, "characteristics.samples_per_second.normalized_per_core": 0.2395, "characteristics.samples_per_second.normalized_per_processor": 28.74, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite3-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "8b7213a7ca84f604", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 11700949, "90.00 percentile latency (ns)": 14226560, "90th percentile latency (ns)": 14226560, "95.00 percentile latency (ns)": 14928734, "97.00 percentile latency (ns)": 15892207, "99.00 percentile latency (ns)": 19714517, "99.90 percentile latency (ns)": 26789574, "Max latency (ns)": 38274528, "Mean latency (ns)": 11396542, "Min duration satisfied": "Yes", "Min latency (ns)": 8118944, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 87.64, "QPS w/o loadgen overhead": 87.75, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.216, "characteristics.good": 33108, "characteristics.power": 14.648716666666656, "characteristics.power.normalized_per_core": 0.12207263888888881, "characteristics.power.normalized_per_processor": 14.648716666666656, "characteristics.samples_per_second": 87.75, "characteristics.samples_per_second.normalized_per_core": 0.73125, "characteristics.samples_per_second.normalized_per_processor": 87.75, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-small_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "dbf00eed61524670", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 87686447, "90.00 percentile latency (ns)": 104488814, "90th percentile latency (ns)": 104488814, "95.00 percentile latency (ns)": 110928094, "97.00 percentile latency (ns)": 118204491, "99.00 percentile latency (ns)": 134703492, "99.90 percentile latency (ns)": 149362659, "Max latency (ns)": 162126919, "Mean latency (ns)": 92041210, "Min duration satisfied": "Yes", "Min latency (ns)": 85916368, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 10.86, "QPS w/o loadgen overhead": 10.86, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.348, "characteristics.good": 38174, "characteristics.power": 20.150583333333334, "characteristics.power.normalized_per_core": 0.16792152777777777, "characteristics.power.normalized_per_processor": 20.150583333333334, "characteristics.samples_per_second": 10.86, "characteristics.samples_per_second.normalized_per_core": 0.0905, "characteristics.samples_per_second.normalized_per_processor": 10.86, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite3-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "7199d483cbf34db3", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 10295678, "90.00 percentile latency (ns)": 17612079, "90th percentile latency (ns)": 17612079, "95.00 percentile latency (ns)": 19377658, "97.00 percentile latency (ns)": 20335198, "99.00 percentile latency (ns)": 21876453, "99.90 percentile latency (ns)": 27738730, "Max latency (ns)": 39381539, "Mean latency (ns)": 11158406, "Min duration satisfied": "Yes", "Min latency (ns)": 6066265, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 89.48, "QPS w/o loadgen overhead": 89.62, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 54.504, "characteristics.good": 27252, "characteristics.power": 10.790083333333326, "characteristics.power.normalized_per_core": 0.08991736111111105, "characteristics.power.normalized_per_processor": 10.790083333333326, "characteristics.samples_per_second": 89.62, "characteristics.samples_per_second.normalized_per_core": 0.7468333333333333, "characteristics.samples_per_second.normalized_per_processor": 89.62, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6587c4b8ee7422a9", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 4850387, "90.00 percentile latency (ns)": 7905178, "90th percentile latency (ns)": 7905178, "95.00 percentile latency (ns)": 8778825, "97.00 percentile latency (ns)": 9332885, "99.00 percentile latency (ns)": 10518793, "99.90 percentile latency (ns)": 16128796, "Max latency (ns)": 25272009, "Mean latency (ns)": 5305515, "Min duration satisfied": "Yes", "Min latency (ns)": 2809721, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 187.98, "QPS w/o loadgen overhead": 188.48, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.478, "characteristics.good": 24739, "characteristics.power": 11.085049999999997, "characteristics.power.normalized_per_core": 0.09237541666666664, "characteristics.power.normalized_per_processor": 11.085049999999997, "characteristics.samples_per_second": 188.48, "characteristics.samples_per_second.normalized_per_core": 1.5706666666666667, "characteristics.samples_per_second.normalized_per_processor": 188.48, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "a2efe4bef9b5b112", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 12600869, "90.00 percentile latency (ns)": 20808840, "90th percentile latency (ns)": 20808840, "95.00 percentile latency (ns)": 22081478, "97.00 percentile latency (ns)": 23060815, "99.00 percentile latency (ns)": 25536764, "99.90 percentile latency (ns)": 31097420, "Max latency (ns)": 42036796, "Mean latency (ns)": 14913722, "Min duration satisfied": "Yes", "Min latency (ns)": 11160210, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 67.0, "QPS w/o loadgen overhead": 67.05, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.124, "characteristics.good": 34062, "characteristics.power": 16.114666666666658, "characteristics.power.normalized_per_core": 0.13428888888888882, "characteristics.power.normalized_per_processor": 16.114666666666658, "characteristics.samples_per_second": 67.05, "characteristics.samples_per_second.normalized_per_core": 0.55875, "characteristics.samples_per_second.normalized_per_processor": 67.05, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "715490595b761a03", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 14847537, "90.00 percentile latency (ns)": 23767686, "90th percentile latency (ns)": 23767686, "95.00 percentile latency (ns)": 25586518, "97.00 percentile latency (ns)": 27621324, "99.00 percentile latency (ns)": 31785471, "99.90 percentile latency (ns)": 38084326, "Max latency (ns)": 49048020, "Mean latency (ns)": 18236892, "Min duration satisfied": "Yes", "Min latency (ns)": 13029893, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 54.79, "QPS w/o loadgen overhead": 54.83, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.76, "characteristics.good": 37880, "characteristics.power": 16.504666666666665, "characteristics.power.normalized_per_core": 0.13753888888888888, "characteristics.power.normalized_per_processor": 16.504666666666665, "characteristics.samples_per_second": 54.83, "characteristics.samples_per_second.normalized_per_core": 0.45691666666666664, "characteristics.samples_per_second.normalized_per_processor": 54.83, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-large_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "3c2fc46a7859f63f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 11799930, "90.00 percentile latency (ns)": 20075981, "90th percentile latency (ns)": 20075981, "95.00 percentile latency (ns)": 21976575, "97.00 percentile latency (ns)": 23093602, "99.00 percentile latency (ns)": 25017296, "99.90 percentile latency (ns)": 30699357, "Max latency (ns)": 48894558, "Mean latency (ns)": 12554551, "Min duration satisfied": "Yes", "Min latency (ns)": 6036492, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 79.51, "QPS w/o loadgen overhead": 79.65, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 65.75, "characteristics.good": 32875, "characteristics.power": 11.59743333333333, "characteristics.power.normalized_per_core": 0.09664527777777775, "characteristics.power.normalized_per_processor": 11.59743333333333, "characteristics.samples_per_second": 79.65, "characteristics.samples_per_second.normalized_per_core": 0.6637500000000001, "characteristics.samples_per_second.normalized_per_processor": 79.65, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4293431e45549dd6", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 12408211, "90.00 percentile latency (ns)": 14734168, "90th percentile latency (ns)": 14734168, "95.00 percentile latency (ns)": 15848200, "97.00 percentile latency (ns)": 16659306, "99.00 percentile latency (ns)": 19002830, "99.90 percentile latency (ns)": 24906860, "Max latency (ns)": 32794974, "Mean latency (ns)": 12467659, "Min duration satisfied": "Yes", "Min latency (ns)": 7373053, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 80.13, "QPS w/o loadgen overhead": 80.21, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.766, "characteristics.good": 32383, "characteristics.power": 13.87046666666668, "characteristics.power.normalized_per_core": 0.11558722222222233, "characteristics.power.normalized_per_processor": 13.87046666666668, "characteristics.samples_per_second": 80.21, "characteristics.samples_per_second.normalized_per_core": 0.6684166666666667, "characteristics.samples_per_second.normalized_per_processor": 80.21, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "da35471943e3beb6", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 13889707, "90.00 percentile latency (ns)": 19743511, "90th percentile latency (ns)": 19743511, "95.00 percentile latency (ns)": 21708109, "97.00 percentile latency (ns)": 23020953, "99.00 percentile latency (ns)": 25914324, "99.90 percentile latency (ns)": 32189641, "Max latency (ns)": 40929992, "Mean latency (ns)": 14088362, "Min duration satisfied": "Yes", "Min latency (ns)": 8254116, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 70.89, "QPS w/o loadgen overhead": 70.98, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.122, "characteristics.good": 35061, "characteristics.power": 14.905483333333338, "characteristics.power.normalized_per_core": 0.12421236111111116, "characteristics.power.normalized_per_processor": 14.905483333333338, "characteristics.samples_per_second": 70.98, "characteristics.samples_per_second.normalized_per_core": 0.5915, "characteristics.samples_per_second.normalized_per_processor": 70.98, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "67b4fdea9ff29f3e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 10883087, "90.00 percentile latency (ns)": 12784177, "90th percentile latency (ns)": 12784177, "95.00 percentile latency (ns)": 13809394, "97.00 percentile latency (ns)": 14669286, "99.00 percentile latency (ns)": 17163106, "99.90 percentile latency (ns)": 23151734, "Max latency (ns)": 26638585, "Mean latency (ns)": 10170743, "Min duration satisfied": "Yes", "Min latency (ns)": 6828408, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 98.23, "QPS w/o loadgen overhead": 98.32, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.128, "characteristics.good": 32064, "characteristics.power": 14.684016666666666, "characteristics.power.normalized_per_core": 0.12236680555555556, "characteristics.power.normalized_per_processor": 14.684016666666666, "characteristics.samples_per_second": 98.32, "characteristics.samples_per_second.normalized_per_core": 0.8193333333333332, "characteristics.samples_per_second.normalized_per_processor": 98.32, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "717abd4f1a91ad4a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 9645589, "90.00 percentile latency (ns)": 17905724, "90th percentile latency (ns)": 17905724, "95.00 percentile latency (ns)": 19645245, "97.00 percentile latency (ns)": 20643895, "99.00 percentile latency (ns)": 22260366, "99.90 percentile latency (ns)": 27634674, "Max latency (ns)": 37594544, "Mean latency (ns)": 10766854, "Min duration satisfied": "Yes", "Min latency (ns)": 5336570, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 92.7, "QPS w/o loadgen overhead": 92.88, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.282, "characteristics.good": 32141, "characteristics.power": 11.368883333333331, "characteristics.power.normalized_per_core": 0.09474069444444443, "characteristics.power.normalized_per_processor": 11.368883333333331, "characteristics.samples_per_second": 92.88, "characteristics.samples_per_second.normalized_per_core": 0.7739999999999999, "characteristics.samples_per_second.normalized_per_processor": 92.88, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "34e45ba94fe11bf4", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 12386914, "90.00 percentile latency (ns)": 15087873, "90th percentile latency (ns)": 15087873, "95.00 percentile latency (ns)": 15962956, "97.00 percentile latency (ns)": 16917449, "99.00 percentile latency (ns)": 20194603, "99.90 percentile latency (ns)": 26137350, "Max latency (ns)": 35598802, "Mean latency (ns)": 12353549, "Min duration satisfied": "Yes", "Min latency (ns)": 7862671, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 80.84, "QPS w/o loadgen overhead": 80.95, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.678, "characteristics.good": 36339, "characteristics.power": 15.869333333333334, "characteristics.power.normalized_per_core": 0.13224444444444444, "characteristics.power.normalized_per_processor": 15.869333333333334, "characteristics.samples_per_second": 80.95, "characteristics.samples_per_second.normalized_per_core": 0.6745833333333333, "characteristics.samples_per_second.normalized_per_processor": 80.95, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "b8c645c69921a641", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 12596121, "90.00 percentile latency (ns)": 15290958, "90th percentile latency (ns)": 15290958, "95.00 percentile latency (ns)": 16760870, "97.00 percentile latency (ns)": 18043262, "99.00 percentile latency (ns)": 21356477, "99.90 percentile latency (ns)": 27168513, "Max latency (ns)": 40830782, "Mean latency (ns)": 12029284, "Min duration satisfied": "Yes", "Min latency (ns)": 8415734, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 83.04, "QPS w/o loadgen overhead": 83.13, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 65.004, "characteristics.good": 32502, "characteristics.power": 14.380466666666667, "characteristics.power.normalized_per_core": 0.11983722222222222, "characteristics.power.normalized_per_processor": 14.380466666666667, "characteristics.samples_per_second": 83.13, "characteristics.samples_per_second.normalized_per_core": 0.69275, "characteristics.samples_per_second.normalized_per_processor": 83.13, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "a925b4bb78926be3", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 14655680, "90.00 percentile latency (ns)": 19421984, "90th percentile latency (ns)": 19421984, "95.00 percentile latency (ns)": 21406057, "97.00 percentile latency (ns)": 22819220, "99.00 percentile latency (ns)": 25854705, "99.90 percentile latency (ns)": 31995960, "Max latency (ns)": 44052452, "Mean latency (ns)": 14823898, "Min duration satisfied": "Yes", "Min latency (ns)": 9186931, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 67.39, "QPS w/o loadgen overhead": 67.46, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.77, "characteristics.good": 35885, "characteristics.power": 16.210416666666664, "characteristics.power.normalized_per_core": 0.13508680555555552, "characteristics.power.normalized_per_processor": 16.210416666666664, "characteristics.samples_per_second": 67.46, "characteristics.samples_per_second.normalized_per_core": 0.5621666666666666, "characteristics.samples_per_second.normalized_per_processor": 67.46, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "8beaf393e500c04e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 10183070, "90.00 percentile latency (ns)": 17300442, "90th percentile latency (ns)": 17300442, "95.00 percentile latency (ns)": 19605475, "97.00 percentile latency (ns)": 21083219, "99.00 percentile latency (ns)": 23597920, "99.90 percentile latency (ns)": 29291756, "Max latency (ns)": 36881120, "Mean latency (ns)": 10983925, "Min duration satisfied": "Yes", "Min latency (ns)": 5810252, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 90.89, "QPS w/o loadgen overhead": 91.04, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.75, "characteristics.good": 33375, "characteristics.power": 12.861266666666658, "characteristics.power.normalized_per_core": 0.10717722222222216, "characteristics.power.normalized_per_processor": 12.861266666666658, "characteristics.samples_per_second": 91.04, "characteristics.samples_per_second.normalized_per_core": 0.7586666666666667, "characteristics.samples_per_second.normalized_per_processor": 91.04, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "afcc65be343447a9", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 6093970, "90.00 percentile latency (ns)": 10891093, "90th percentile latency (ns)": 10891093, "95.00 percentile latency (ns)": 11635021, "97.00 percentile latency (ns)": 12051141, "99.00 percentile latency (ns)": 12974913, "99.90 percentile latency (ns)": 20249160, "Max latency (ns)": 39859328, "Mean latency (ns)": 6777900, "Min duration satisfied": "Yes", "Min latency (ns)": 3185683, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 147.09, "QPS w/o loadgen overhead": 147.54, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 47.14, "characteristics.good": 23570, "characteristics.power": 9.971066666666658, "characteristics.power.normalized_per_core": 0.08309222222222215, "characteristics.power.normalized_per_processor": 9.971066666666658, "characteristics.samples_per_second": 147.54, "characteristics.samples_per_second.normalized_per_core": 1.2295, "characteristics.samples_per_second.normalized_per_processor": 147.54, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4b3e8b4e131f1903", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 7857212, "90.00 percentile latency (ns)": 13487044, "90th percentile latency (ns)": 13487044, "95.00 percentile latency (ns)": 15007938, "97.00 percentile latency (ns)": 15813474, "99.00 percentile latency (ns)": 17269725, "99.90 percentile latency (ns)": 23649727, "Max latency (ns)": 32825460, "Mean latency (ns)": 8684085, "Min duration satisfied": "Yes", "Min latency (ns)": 4324721, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 114.88, "QPS w/o loadgen overhead": 115.15, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.988, "characteristics.good": 28494, "characteristics.power": 10.805933333333334, "characteristics.power.normalized_per_core": 0.09004944444444445, "characteristics.power.normalized_per_processor": 10.805933333333334, "characteristics.samples_per_second": 115.15, "characteristics.samples_per_second.normalized_per_core": 0.9595833333333333, "characteristics.samples_per_second.normalized_per_processor": 115.15, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4919ab8363f7af37", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 6050816, "90.00 percentile latency (ns)": 9564182, "90th percentile latency (ns)": 9564182, "95.00 percentile latency (ns)": 10878509, "97.00 percentile latency (ns)": 11809780, "99.00 percentile latency (ns)": 13629473, "99.90 percentile latency (ns)": 19156490, "Max latency (ns)": 27042139, "Mean latency (ns)": 6402536, "Min duration satisfied": "Yes", "Min latency (ns)": 3183430, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 155.76, "QPS w/o loadgen overhead": 156.19, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.716, "characteristics.good": 31858, "characteristics.power": 13.058166666666677, "characteristics.power.normalized_per_core": 0.10881805555555564, "characteristics.power.normalized_per_processor": 13.058166666666677, "characteristics.samples_per_second": 156.19, "characteristics.samples_per_second.normalized_per_core": 1.3015833333333333, "characteristics.samples_per_second.normalized_per_processor": 156.19, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "27214416fa892b66", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 9420405, "90.00 percentile latency (ns)": 16005155, "90th percentile latency (ns)": 16005155, "95.00 percentile latency (ns)": 17550499, "97.00 percentile latency (ns)": 18343318, "99.00 percentile latency (ns)": 19564749, "99.90 percentile latency (ns)": 25011149, "Max latency (ns)": 34015626, "Mean latency (ns)": 10219255, "Min duration satisfied": "Yes", "Min latency (ns)": 5425590, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 97.69, "QPS w/o loadgen overhead": 97.85, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 55.51, "characteristics.good": 27755, "characteristics.power": 10.37155000000001, "characteristics.power.normalized_per_core": 0.08642958333333342, "characteristics.power.normalized_per_processor": 10.37155000000001, "characteristics.samples_per_second": 97.85, "characteristics.samples_per_second.normalized_per_core": 0.8154166666666666, "characteristics.samples_per_second.normalized_per_processor": 97.85, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "588d43c521c62c3a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 8415385, "90.00 percentile latency (ns)": 14793222, "90th percentile latency (ns)": 14793222, "95.00 percentile latency (ns)": 16483995, "97.00 percentile latency (ns)": 17565591, "99.00 percentile latency (ns)": 19298008, "99.90 percentile latency (ns)": 25717747, "Max latency (ns)": 35524418, "Mean latency (ns)": 9349264, "Min duration satisfied": "Yes", "Min latency (ns)": 4930188, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 106.74, "QPS w/o loadgen overhead": 106.96, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.322, "characteristics.good": 29661, "characteristics.power": 11.290033333333321, "characteristics.power.normalized_per_core": 0.094083611111111, "characteristics.power.normalized_per_processor": 11.290033333333321, "characteristics.samples_per_second": 106.96, "characteristics.samples_per_second.normalized_per_core": 0.8913333333333333, "characteristics.samples_per_second.normalized_per_processor": 106.96, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "fc8bda20184055b1", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 5935910, "90.00 percentile latency (ns)": 8821082, "90th percentile latency (ns)": 8821082, "95.00 percentile latency (ns)": 10439943, "97.00 percentile latency (ns)": 11471101, "99.00 percentile latency (ns)": 14846180, "99.90 percentile latency (ns)": 63162865, "Max latency (ns)": 77892953, "Mean latency (ns)": 6800045, "Min duration satisfied": "Yes", "Min latency (ns)": 5453908, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 146.65, "QPS w/o loadgen overhead": 147.06, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.242, "characteristics.good": 34121, "characteristics.power": 25.71076666666666, "characteristics.power.normalized_per_core": 0.21425638888888884, "characteristics.power.normalized_per_processor": 25.71076666666666, "characteristics.samples_per_second": 147.06, "characteristics.samples_per_second.normalized_per_core": 1.2255, "characteristics.samples_per_second.normalized_per_processor": 147.06, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "59b471f750af4ff9", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7387225, "90.00 percentile latency (ns)": 11862205, "90th percentile latency (ns)": 11862205, "95.00 percentile latency (ns)": 13461935, "97.00 percentile latency (ns)": 14389471, "99.00 percentile latency (ns)": 21674194, "99.90 percentile latency (ns)": 80755131, "Max latency (ns)": 134672284, "Mean latency (ns)": 8704766, "Min duration satisfied": "Yes", "Min latency (ns)": 6555471, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 114.61, "QPS w/o loadgen overhead": 114.88, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.738, "characteristics.good": 35369, "characteristics.power": 25.14968333333331, "characteristics.power.normalized_per_core": 0.20958069444444424, "characteristics.power.normalized_per_processor": 25.14968333333331, "characteristics.samples_per_second": 114.88, "characteristics.samples_per_second.normalized_per_core": 0.9573333333333333, "characteristics.samples_per_second.normalized_per_processor": 114.88, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "a32ce4427a7af657", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 75243435, "90.00 percentile latency (ns)": 86725299, "90th percentile latency (ns)": 86725299, "95.00 percentile latency (ns)": 89973198, "97.00 percentile latency (ns)": 93919483, "99.00 percentile latency (ns)": 109025993, "99.90 percentile latency (ns)": 152441451, "Max latency (ns)": 167181661, "Mean latency (ns)": 78134307, "Min duration satisfied": "Yes", "Min latency (ns)": 68546254, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 12.8, "QPS w/o loadgen overhead": 12.8, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 78.514, "characteristics.good": 39257, "characteristics.power": 21.245540765391027, "characteristics.power.normalized_per_core": 0.17704617304492523, "characteristics.power.normalized_per_processor": 21.245540765391027, "characteristics.samples_per_second": 12.8, "characteristics.samples_per_second.normalized_per_core": 0.10666666666666667, "characteristics.samples_per_second.normalized_per_processor": 12.8, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite4-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "346418c1393ee1dd", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6774996, "90.00 percentile latency (ns)": 10172041, "90th percentile latency (ns)": 10172041, "95.00 percentile latency (ns)": 12683409, "97.00 percentile latency (ns)": 13761691, "99.00 percentile latency (ns)": 16649121, "99.90 percentile latency (ns)": 68859475, "Max latency (ns)": 132311713, "Mean latency (ns)": 7716165, "Min duration satisfied": "Yes", "Min latency (ns)": 5969775, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 129.44, "QPS w/o loadgen overhead": 129.6, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.542, "characteristics.good": 31771, "characteristics.power": 20.319666666666706, "characteristics.power.normalized_per_core": 0.16933055555555587, "characteristics.power.normalized_per_processor": 20.319666666666706, "characteristics.samples_per_second": 129.6, "characteristics.samples_per_second.normalized_per_core": 1.0799999999999998, "characteristics.samples_per_second.normalized_per_processor": 129.6, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6a3c5a7792e4d6bb", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2004023, "90.00 percentile latency (ns)": 2303044, "90th percentile latency (ns)": 2303044, "95.00 percentile latency (ns)": 2830940, "97.00 percentile latency (ns)": 3460407, "99.00 percentile latency (ns)": 5508911, "99.90 percentile latency (ns)": 24756120, "Max latency (ns)": 138200814, "Mean latency (ns)": 2206973, "Min duration satisfied": "Yes", "Min latency (ns)": 1673640, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 451.37, "QPS w/o loadgen overhead": 453.11, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 52.172, "characteristics.good": 26086, "characteristics.power": 22.605016666666682, "characteristics.power.normalized_per_core": 0.18837513888888902, "characteristics.power.normalized_per_processor": 22.605016666666682, "characteristics.samples_per_second": 453.11, "characteristics.samples_per_second.normalized_per_core": 3.775916666666667, "characteristics.samples_per_second.normalized_per_processor": 453.11, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4748b2b82a1bd829", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2312751, "90.00 percentile latency (ns)": 2690979, "90th percentile latency (ns)": 2690979, "95.00 percentile latency (ns)": 3341601, "97.00 percentile latency (ns)": 4109281, "99.00 percentile latency (ns)": 6640792, "99.90 percentile latency (ns)": 27254171, "Max latency (ns)": 135076480, "Mean latency (ns)": 2559402, "Min duration satisfied": "Yes", "Min latency (ns)": 1979805, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 388.89, "QPS w/o loadgen overhead": 390.72, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.402, "characteristics.good": 29201, "characteristics.power": 23.124483333333362, "characteristics.power.normalized_per_core": 0.192704027777778, "characteristics.power.normalized_per_processor": 23.124483333333362, "characteristics.samples_per_second": 390.72, "characteristics.samples_per_second.normalized_per_core": 3.2560000000000002, "characteristics.samples_per_second.normalized_per_processor": 390.72, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "48aeab4b5beb6505", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4347462, "90.00 percentile latency (ns)": 5140074, "90th percentile latency (ns)": 5140074, "95.00 percentile latency (ns)": 6157370, "97.00 percentile latency (ns)": 7285293, "99.00 percentile latency (ns)": 10902964, "99.90 percentile latency (ns)": 46317558, "Max latency (ns)": 142162752, "Mean latency (ns)": 4747547, "Min duration satisfied": "Yes", "Min latency (ns)": 3890001, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 209.84, "QPS w/o loadgen overhead": 210.64, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 60.976, "characteristics.good": 30488, "characteristics.power": 23.757699999999986, "characteristics.power.normalized_per_core": 0.19798083333333322, "characteristics.power.normalized_per_processor": 23.757699999999986, "characteristics.samples_per_second": 210.64, "characteristics.samples_per_second.normalized_per_core": 1.7553333333333332, "characteristics.samples_per_second.normalized_per_processor": 210.64, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d8575433e8eaa377", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2678012, "90.00 percentile latency (ns)": 3148048, "90th percentile latency (ns)": 3148048, "95.00 percentile latency (ns)": 3924948, "97.00 percentile latency (ns)": 4699321, "99.00 percentile latency (ns)": 7627039, "99.90 percentile latency (ns)": 35090875, "Max latency (ns)": 75700786, "Mean latency (ns)": 2948521, "Min duration satisfied": "Yes", "Min latency (ns)": 2251909, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 338.44, "QPS w/o loadgen overhead": 339.15, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 48.162, "characteristics.good": 24081, "characteristics.power": 19.785566666666664, "characteristics.power.normalized_per_core": 0.1648797222222222, "characteristics.power.normalized_per_processor": 19.785566666666664, "characteristics.samples_per_second": 339.15, "characteristics.samples_per_second.normalized_per_core": 2.82625, "characteristics.samples_per_second.normalized_per_processor": 339.15, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 500, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "aa795457cf404cbb", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 12049130, "90.00 percentile latency (ns)": 14287235, "90th percentile latency (ns)": 14287235, "95.00 percentile latency (ns)": 18307725, "97.00 percentile latency (ns)": 19320947, "99.00 percentile latency (ns)": 26441360, "99.90 percentile latency (ns)": 68461158, "Max latency (ns)": 117024350, "Mean latency (ns)": 12725958, "Min duration satisfied": "Yes", "Min latency (ns)": 9931690, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 78.46, "QPS w/o loadgen overhead": 78.58, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.068, "characteristics.good": 37534, "characteristics.power": 26.56621666666668, "characteristics.power.normalized_per_core": 0.221385138888889, "characteristics.power.normalized_per_processor": 26.56621666666668, "characteristics.samples_per_second": 78.58, "characteristics.samples_per_second.normalized_per_core": 0.6548333333333333, "characteristics.samples_per_second.normalized_per_processor": 78.58, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite0-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "2b1e5a57e5627b83", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3086158, "90.00 percentile latency (ns)": 3818800, "90th percentile latency (ns)": 3818800, "95.00 percentile latency (ns)": 4862211, "97.00 percentile latency (ns)": 5905551, "99.00 percentile latency (ns)": 8764824, "99.90 percentile latency (ns)": 42552336, "Max latency (ns)": 80273167, "Mean latency (ns)": 3402604, "Min duration satisfied": "Yes", "Min latency (ns)": 2585241, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 293.41, "QPS w/o loadgen overhead": 293.89, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 50.242, "characteristics.good": 25121, "characteristics.power": 20.010683333333326, "characteristics.power.normalized_per_core": 0.16675569444444438, "characteristics.power.normalized_per_processor": 20.010683333333326, "characteristics.samples_per_second": 293.89, "characteristics.samples_per_second.normalized_per_core": 2.4490833333333333, "characteristics.samples_per_second.normalized_per_processor": 293.89, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 500, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4b46804415cad5fc", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2825661, "90.00 percentile latency (ns)": 3094634, "90th percentile latency (ns)": 3094634, "95.00 percentile latency (ns)": 3338741, "97.00 percentile latency (ns)": 4025266, "99.00 percentile latency (ns)": 6713771, "99.90 percentile latency (ns)": 30102933, "Max latency (ns)": 88748386, "Mean latency (ns)": 3032182, "Min duration satisfied": "Yes", "Min latency (ns)": 2574322, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 328.37, "QPS w/o loadgen overhead": 329.8, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 60.842, "characteristics.good": 30421, "characteristics.power": 24.847666666666683, "characteristics.power.normalized_per_core": 0.20706388888888902, "characteristics.power.normalized_per_processor": 24.847666666666683, "characteristics.samples_per_second": 329.8, "characteristics.samples_per_second.normalized_per_core": 2.7483333333333335, "characteristics.samples_per_second.normalized_per_processor": 329.8, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c4ae684ffe486397", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5749243, "90.00 percentile latency (ns)": 6680630, "90th percentile latency (ns)": 6680630, "95.00 percentile latency (ns)": 7778289, "97.00 percentile latency (ns)": 9178566, "99.00 percentile latency (ns)": 13542223, "99.90 percentile latency (ns)": 67815831, "Max latency (ns)": 129020392, "Mean latency (ns)": 6237201, "Min duration satisfied": "Yes", "Min latency (ns)": 5069019, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 159.86, "QPS w/o loadgen overhead": 160.33, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.016, "characteristics.good": 34508, "characteristics.power": 25.460249999999995, "characteristics.power.normalized_per_core": 0.21216874999999996, "characteristics.power.normalized_per_processor": 25.460249999999995, "characteristics.samples_per_second": 160.33, "characteristics.samples_per_second.normalized_per_core": 1.3360833333333335, "characteristics.samples_per_second.normalized_per_processor": 160.33, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "da64f83a933c395b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 21024352, "90.00 percentile latency (ns)": 29427836, "90th percentile latency (ns)": 29427836, "95.00 percentile latency (ns)": 31514149, "97.00 percentile latency (ns)": 33240893, "99.00 percentile latency (ns)": 41130339, "99.90 percentile latency (ns)": 88365544, "Max latency (ns)": 123857685, "Mean latency (ns)": 23467361, "Min duration satisfied": "Yes", "Min latency (ns)": 19119026, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 42.58, "QPS w/o loadgen overhead": 42.61, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.098, "characteristics.good": 37049, "characteristics.power": 20.983200000000014, "characteristics.power.normalized_per_core": 0.17486000000000013, "characteristics.power.normalized_per_processor": 20.983200000000014, "characteristics.samples_per_second": 42.61, "characteristics.samples_per_second.normalized_per_core": 0.3550833333333333, "characteristics.samples_per_second.normalized_per_processor": 42.61, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite1-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "bc7bba9e4365a257", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5403260, "90.00 percentile latency (ns)": 7857300, "90th percentile latency (ns)": 7857300, "95.00 percentile latency (ns)": 9893930, "97.00 percentile latency (ns)": 11460684, "99.00 percentile latency (ns)": 13595318, "99.90 percentile latency (ns)": 67005467, "Max latency (ns)": 145301465, "Mean latency (ns)": 6190484, "Min duration satisfied": "Yes", "Min latency (ns)": 4762398, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 161.32, "QPS w/o loadgen overhead": 161.54, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.924, "characteristics.good": 30962, "characteristics.power": 19.849233333333338, "characteristics.power.normalized_per_core": 0.1654102777777778, "characteristics.power.normalized_per_processor": 19.849233333333338, "characteristics.samples_per_second": 161.54, "characteristics.samples_per_second.normalized_per_core": 1.3461666666666665, "characteristics.samples_per_second.normalized_per_processor": 161.54, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "676fd4b0c862ff62", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5363837, "90.00 percentile latency (ns)": 7716829, "90th percentile latency (ns)": 7716829, "95.00 percentile latency (ns)": 10594927, "97.00 percentile latency (ns)": 11688218, "99.00 percentile latency (ns)": 13198615, "99.90 percentile latency (ns)": 31086166, "Max latency (ns)": 75871605, "Mean latency (ns)": 6051837, "Min duration satisfied": "Yes", "Min latency (ns)": 4992792, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 165.05, "QPS w/o loadgen overhead": 165.24, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 62.298, "characteristics.good": 31149, "characteristics.power": 20.24790000000004, "characteristics.power.normalized_per_core": 0.16873250000000034, "characteristics.power.normalized_per_processor": 20.24790000000004, "characteristics.samples_per_second": 165.24, "characteristics.samples_per_second.normalized_per_core": 1.377, "characteristics.samples_per_second.normalized_per_processor": 165.24, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "5f0145f42b41bb9d", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2733655, "90.00 percentile latency (ns)": 3013443, "90th percentile latency (ns)": 3013443, "95.00 percentile latency (ns)": 3697602, "97.00 percentile latency (ns)": 4309789, "99.00 percentile latency (ns)": 6514749, "99.90 percentile latency (ns)": 29527467, "Max latency (ns)": 117642041, "Mean latency (ns)": 2970037, "Min duration satisfied": "Yes", "Min latency (ns)": 2353831, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 335.94, "QPS w/o loadgen overhead": 336.7, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.12, "characteristics.good": 24560, "characteristics.power": 19.43801666666667, "characteristics.power.normalized_per_core": 0.16198347222222226, "characteristics.power.normalized_per_processor": 19.43801666666667, "characteristics.samples_per_second": 336.7, "characteristics.samples_per_second.normalized_per_core": 2.805833333333333, "characteristics.samples_per_second.normalized_per_processor": 336.7, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "159e654f94813745", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 22366278, "90.00 percentile latency (ns)": 29945726, "90th percentile latency (ns)": 29945726, "95.00 percentile latency (ns)": 32811952, "97.00 percentile latency (ns)": 36115874, "99.00 percentile latency (ns)": 45489653, "99.90 percentile latency (ns)": 121180285, "Max latency (ns)": 144259273, "Mean latency (ns)": 24517861, "Min duration satisfied": "Yes", "Min latency (ns)": 18789759, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 40.76, "QPS w/o loadgen overhead": 40.79, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.584, "characteristics.good": 36792, "characteristics.power": 20.35913333333335, "characteristics.power.normalized_per_core": 0.16965944444444458, "characteristics.power.normalized_per_processor": 20.35913333333335, "characteristics.samples_per_second": 40.79, "characteristics.samples_per_second.normalized_per_core": 0.33991666666666664, "characteristics.samples_per_second.normalized_per_processor": 40.79, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.4-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.4-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_140.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6997313afd485cb7", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 15252679, "90.00 percentile latency (ns)": 16077069, "90th percentile latency (ns)": 16077069, "95.00 percentile latency (ns)": 20072509, "97.00 percentile latency (ns)": 21181903, "99.00 percentile latency (ns)": 28763583, "99.90 percentile latency (ns)": 62522182, "Max latency (ns)": 92075146, "Mean latency (ns)": 15886903, "Min duration satisfied": "Yes", "Min latency (ns)": 13921162, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 62.89, "QPS w/o loadgen overhead": 62.94, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.772, "characteristics.good": 35386, "characteristics.power": 20.56610000000001, "characteristics.power.normalized_per_core": 0.17138416666666675, "characteristics.power.normalized_per_processor": 20.56610000000001, "characteristics.samples_per_second": 62.94, "characteristics.samples_per_second.normalized_per_core": 0.5245, "characteristics.samples_per_second.normalized_per_processor": 62.94, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "b5444aac4df4f4fe", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 1797659, "90.00 percentile latency (ns)": 1980312, "90th percentile latency (ns)": 1980312, "95.00 percentile latency (ns)": 2217748, "97.00 percentile latency (ns)": 2738703, "99.00 percentile latency (ns)": 3926381, "99.90 percentile latency (ns)": 24750498, "Max latency (ns)": 79677783, "Mean latency (ns)": 1946044, "Min duration satisfied": "Yes", "Min latency (ns)": 1534979, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 512.4, "QPS w/o loadgen overhead": 513.86, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 47.808, "characteristics.good": 23904, "characteristics.power": 22.638133333333336, "characteristics.power.normalized_per_core": 0.18865111111111113, "characteristics.power.normalized_per_processor": 22.638133333333336, "characteristics.samples_per_second": 513.86, "characteristics.samples_per_second.normalized_per_core": 4.282166666666667, "characteristics.samples_per_second.normalized_per_processor": 513.86, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 500, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "b1acb1495b9af479", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3559861, "90.00 percentile latency (ns)": 5586185, "90th percentile latency (ns)": 5586185, "95.00 percentile latency (ns)": 7035301, "97.00 percentile latency (ns)": 8251032, "99.00 percentile latency (ns)": 10847462, "99.90 percentile latency (ns)": 49331373, "Max latency (ns)": 141421723, "Mean latency (ns)": 4196199, "Min duration satisfied": "Yes", "Min latency (ns)": 2889305, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 237.22, "QPS w/o loadgen overhead": 238.31, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.038, "characteristics.good": 32019, "characteristics.power": 24.03178333333334, "characteristics.power.normalized_per_core": 0.20026486111111116, "characteristics.power.normalized_per_processor": 24.03178333333334, "characteristics.samples_per_second": 238.31, "characteristics.samples_per_second.normalized_per_core": 1.9859166666666668, "characteristics.samples_per_second.normalized_per_processor": 238.31, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "7c4534b8198d83a6", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2975319, "90.00 percentile latency (ns)": 3358053, "90th percentile latency (ns)": 3358053, "95.00 percentile latency (ns)": 4072648, "97.00 percentile latency (ns)": 4849000, "99.00 percentile latency (ns)": 7074087, "99.90 percentile latency (ns)": 32321754, "Max latency (ns)": 88145684, "Mean latency (ns)": 3234955, "Min duration satisfied": "Yes", "Min latency (ns)": 2635076, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 308.4, "QPS w/o loadgen overhead": 309.12, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 51.848, "characteristics.good": 25924, "characteristics.power": 23.123183333333333, "characteristics.power.normalized_per_core": 0.19269319444444444, "characteristics.power.normalized_per_processor": 23.123183333333333, "characteristics.samples_per_second": 309.12, "characteristics.samples_per_second.normalized_per_core": 2.576, "characteristics.samples_per_second.normalized_per_processor": 309.12, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 500, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d8647438b0f6c96d", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4763392, "90.00 percentile latency (ns)": 5540407, "90th percentile latency (ns)": 5540407, "95.00 percentile latency (ns)": 6670829, "97.00 percentile latency (ns)": 7855555, "99.00 percentile latency (ns)": 11655924, "99.90 percentile latency (ns)": 43443844, "Max latency (ns)": 140267059, "Mean latency (ns)": 5145735, "Min duration satisfied": "Yes", "Min latency (ns)": 3944274, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 194.02, "QPS w/o loadgen overhead": 194.34, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.226, "characteristics.good": 29613, "characteristics.power": 19.623666666666697, "characteristics.power.normalized_per_core": 0.16353055555555582, "characteristics.power.normalized_per_processor": 19.623666666666697, "characteristics.samples_per_second": 194.34, "characteristics.samples_per_second.normalized_per_core": 1.6195, "characteristics.samples_per_second.normalized_per_processor": 194.34, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "e14546bfab45e397", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9034290, "90.00 percentile latency (ns)": 11048492, "90th percentile latency (ns)": 11048492, "95.00 percentile latency (ns)": 13453560, "97.00 percentile latency (ns)": 15148707, "99.00 percentile latency (ns)": 23208907, "99.90 percentile latency (ns)": 67778382, "Max latency (ns)": 85158901, "Mean latency (ns)": 9800484, "Min duration satisfied": "Yes", "Min latency (ns)": 8152619, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 101.93, "QPS w/o loadgen overhead": 102.04, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.248, "characteristics.good": 34124, "characteristics.power": 20.309566666666687, "characteristics.power.normalized_per_core": 0.16924638888888904, "characteristics.power.normalized_per_processor": 20.309566666666687, "characteristics.samples_per_second": 102.04, "characteristics.samples_per_second.normalized_per_core": 0.8503333333333334, "characteristics.samples_per_second.normalized_per_processor": 102.04, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "5adbc0183e464e94", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4157071, "90.00 percentile latency (ns)": 5701501, "90th percentile latency (ns)": 5701501, "95.00 percentile latency (ns)": 7414766, "97.00 percentile latency (ns)": 9101941, "99.00 percentile latency (ns)": 11934770, "99.90 percentile latency (ns)": 46184673, "Max latency (ns)": 129686422, "Mean latency (ns)": 4726920, "Min duration satisfied": "Yes", "Min latency (ns)": 3546571, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 211.23, "QPS w/o loadgen overhead": 211.55, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.12, "characteristics.good": 29560, "characteristics.power": 19.676283333333306, "characteristics.power.normalized_per_core": 0.16396902777777755, "characteristics.power.normalized_per_processor": 19.676283333333306, "characteristics.samples_per_second": 211.55, "characteristics.samples_per_second.normalized_per_core": 1.7629166666666667, "characteristics.samples_per_second.normalized_per_processor": 211.55, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "84334b34947f5ccb", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4275801, "90.00 percentile latency (ns)": 5040253, "90th percentile latency (ns)": 5040253, "95.00 percentile latency (ns)": 6141320, "97.00 percentile latency (ns)": 7345539, "99.00 percentile latency (ns)": 10978463, "99.90 percentile latency (ns)": 46601220, "Max latency (ns)": 77244039, "Mean latency (ns)": 4704346, "Min duration satisfied": "Yes", "Min latency (ns)": 3764706, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 212.24, "QPS w/o loadgen overhead": 212.57, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.46, "characteristics.good": 29230, "characteristics.power": 20.236233333333313, "characteristics.power.normalized_per_core": 0.16863527777777762, "characteristics.power.normalized_per_processor": 20.236233333333313, "characteristics.samples_per_second": 212.57, "characteristics.samples_per_second.normalized_per_core": 1.7714166666666666, "characteristics.samples_per_second.normalized_per_processor": 212.57, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "04c183fe104b4a15", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11896158, "90.00 percentile latency (ns)": 17880815, "90th percentile latency (ns)": 17880815, "95.00 percentile latency (ns)": 19362835, "97.00 percentile latency (ns)": 20917369, "99.00 percentile latency (ns)": 29810765, "99.90 percentile latency (ns)": 87706362, "Max latency (ns)": 127367813, "Mean latency (ns)": 13335248, "Min duration satisfied": "Yes", "Min latency (ns)": 10722184, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 74.9, "QPS w/o loadgen overhead": 74.99, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.91, "characteristics.good": 36455, "characteristics.power": 20.723533333333364, "characteristics.power.normalized_per_core": 0.17269611111111136, "characteristics.power.normalized_per_processor": 20.723533333333364, "characteristics.samples_per_second": 74.99, "characteristics.samples_per_second.normalized_per_core": 0.6249166666666667, "characteristics.samples_per_second.normalized_per_processor": 74.99, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-large_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4d86a714e061d90e", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 12990045, "90.00 percentile latency (ns)": 19475194, "90th percentile latency (ns)": 19475194, "95.00 percentile latency (ns)": 20714974, "97.00 percentile latency (ns)": 22275665, "99.00 percentile latency (ns)": 31295369, "99.90 percentile latency (ns)": 91441790, "Max latency (ns)": 127955318, "Mean latency (ns)": 14685748, "Min duration satisfied": "Yes", "Min latency (ns)": 11428565, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 68.04, "QPS w/o loadgen overhead": 68.09, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.542, "characteristics.good": 35271, "characteristics.power": 20.439799999999995, "characteristics.power.normalized_per_core": 0.17033166666666663, "characteristics.power.normalized_per_processor": 20.439799999999995, "characteristics.samples_per_second": 68.09, "characteristics.samples_per_second.normalized_per_core": 0.5674166666666667, "characteristics.samples_per_second.normalized_per_processor": 68.09, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "53542e7749be8bac", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4492967, "90.00 percentile latency (ns)": 5186470, "90th percentile latency (ns)": 5186470, "95.00 percentile latency (ns)": 6061646, "97.00 percentile latency (ns)": 6953301, "99.00 percentile latency (ns)": 10901157, "99.90 percentile latency (ns)": 45914866, "Max latency (ns)": 120802112, "Mean latency (ns)": 4868536, "Min duration satisfied": "Yes", "Min latency (ns)": 4032179, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 205.05, "QPS w/o loadgen overhead": 205.4, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.594, "characteristics.good": 28297, "characteristics.power": 19.667566666666687, "characteristics.power.normalized_per_core": 0.16389638888888905, "characteristics.power.normalized_per_processor": 19.667566666666687, "characteristics.samples_per_second": 205.4, "characteristics.samples_per_second.normalized_per_core": 1.7116666666666667, "characteristics.samples_per_second.normalized_per_processor": 205.4, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "381dcb8027bbcbe2", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7687393, "90.00 percentile latency (ns)": 12927143, "90th percentile latency (ns)": 12927143, "95.00 percentile latency (ns)": 14700340, "97.00 percentile latency (ns)": 15407179, "99.00 percentile latency (ns)": 20440892, "99.90 percentile latency (ns)": 79000104, "Max latency (ns)": 127473706, "Mean latency (ns)": 8832296, "Min duration satisfied": "Yes", "Min latency (ns)": 6603108, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 113.1, "QPS w/o loadgen overhead": 113.22, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.82, "characteristics.good": 33910, "characteristics.power": 20.263933333333345, "characteristics.power.normalized_per_core": 0.16886611111111122, "characteristics.power.normalized_per_processor": 20.263933333333345, "characteristics.samples_per_second": 113.22, "characteristics.samples_per_second.normalized_per_core": 0.9435, "characteristics.samples_per_second.normalized_per_processor": 113.22, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ad1643f4bf095711", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 1352226, "90.00 percentile latency (ns)": 1490246, "90th percentile latency (ns)": 1490246, "95.00 percentile latency (ns)": 1613261, "97.00 percentile latency (ns)": 1972829, "99.00 percentile latency (ns)": 2908395, "99.90 percentile latency (ns)": 21150298, "Max latency (ns)": 74481020, "Mean latency (ns)": 1462121, "Min duration satisfied": "Yes", "Min latency (ns)": 1166743, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 682.05, "QPS w/o loadgen overhead": 683.94, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 44.116, "characteristics.good": 22058, "characteristics.power": 21.658199999999997, "characteristics.power.normalized_per_core": 0.18048499999999998, "characteristics.power.normalized_per_processor": 21.658199999999997, "characteristics.samples_per_second": 683.94, "characteristics.samples_per_second.normalized_per_core": 5.6995000000000005, "characteristics.samples_per_second.normalized_per_processor": 683.94, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 500, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4393948cda70ecbd", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 1887665, "90.00 percentile latency (ns)": 2278209, "90th percentile latency (ns)": 2278209, "95.00 percentile latency (ns)": 3088678, "97.00 percentile latency (ns)": 3843623, "99.00 percentile latency (ns)": 6087306, "99.90 percentile latency (ns)": 28873225, "Max latency (ns)": 76294812, "Mean latency (ns)": 2139917, "Min duration satisfied": "Yes", "Min latency (ns)": 1650791, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 466.54, "QPS w/o loadgen overhead": 467.31, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 45.494, "characteristics.good": 22747, "characteristics.power": 20.16691666666663, "characteristics.power.normalized_per_core": 0.16805763888888858, "characteristics.power.normalized_per_processor": 20.16691666666663, "characteristics.samples_per_second": 467.31, "characteristics.samples_per_second.normalized_per_core": 3.89425, "characteristics.samples_per_second.normalized_per_processor": 467.31, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 500, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "da380733e4a6c830", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9006575, "90.00 percentile latency (ns)": 10921764, "90th percentile latency (ns)": 10921764, "95.00 percentile latency (ns)": 13687999, "97.00 percentile latency (ns)": 15144896, "99.00 percentile latency (ns)": 21287504, "99.90 percentile latency (ns)": 70129956, "Max latency (ns)": 97232185, "Mean latency (ns)": 9986350, "Min duration satisfied": "Yes", "Min latency (ns)": 8223020, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 99.91, "QPS w/o loadgen overhead": 100.14, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.626, "characteristics.good": 35313, "characteristics.power": 26.76593333333332, "characteristics.power.normalized_per_core": 0.22304944444444433, "characteristics.power.normalized_per_processor": 26.76593333333332, "characteristics.samples_per_second": 100.14, "characteristics.samples_per_second.normalized_per_core": 0.8345, "characteristics.samples_per_second.normalized_per_processor": 100.14, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d5c8b58046edbd94", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5975110, "90.00 percentile latency (ns)": 9023916, "90th percentile latency (ns)": 9023916, "95.00 percentile latency (ns)": 11250711, "97.00 percentile latency (ns)": 12575766, "99.00 percentile latency (ns)": 14732254, "99.90 percentile latency (ns)": 70569658, "Max latency (ns)": 121054480, "Mean latency (ns)": 6880608, "Min duration satisfied": "Yes", "Min latency (ns)": 5176086, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 145.12, "QPS w/o loadgen overhead": 145.34, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.454, "characteristics.good": 31727, "characteristics.power": 19.917799999999996, "characteristics.power.normalized_per_core": 0.16598166666666664, "characteristics.power.normalized_per_processor": 19.917799999999996, "characteristics.samples_per_second": 145.34, "characteristics.samples_per_second.normalized_per_core": 1.2111666666666667, "characteristics.samples_per_second.normalized_per_processor": 145.34, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "2288b84190851e41", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11422922, "90.00 percentile latency (ns)": 13050674, "90th percentile latency (ns)": 13050674, "95.00 percentile latency (ns)": 14170082, "97.00 percentile latency (ns)": 16386707, "99.00 percentile latency (ns)": 26609148, "99.90 percentile latency (ns)": 72188969, "Max latency (ns)": 90822857, "Mean latency (ns)": 12091516, "Min duration satisfied": "Yes", "Min latency (ns)": 10337465, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 82.61, "QPS w/o loadgen overhead": 82.7, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.04, "characteristics.good": 35020, "characteristics.power": 20.67551666666663, "characteristics.power.normalized_per_core": 0.17229597222222193, "characteristics.power.normalized_per_processor": 20.67551666666663, "characteristics.samples_per_second": 82.7, "characteristics.samples_per_second.normalized_per_core": 0.6891666666666667, "characteristics.samples_per_second.normalized_per_processor": 82.7, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "348529caddf03223", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 27689468, "90.00 percentile latency (ns)": 40099248, "90th percentile latency (ns)": 40099248, "95.00 percentile latency (ns)": 45296254, "97.00 percentile latency (ns)": 49511439, "99.00 percentile latency (ns)": 53797086, "99.90 percentile latency (ns)": 94966591, "Max latency (ns)": 131743916, "Mean latency (ns)": 31335580, "Min duration satisfied": "Yes", "Min latency (ns)": 25825962, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 31.9, "QPS w/o loadgen overhead": 31.91, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.124, "characteristics.good": 37062, "characteristics.power": 21.0273, "characteristics.power.normalized_per_core": 0.1752275, "characteristics.power.normalized_per_processor": 21.0273, "characteristics.samples_per_second": 31.91, "characteristics.samples_per_second.normalized_per_core": 0.2659166666666667, "characteristics.samples_per_second.normalized_per_processor": 31.91, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite2-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "b7c47d88a4cbdb36", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3362741, "90.00 percentile latency (ns)": 3778665, "90th percentile latency (ns)": 3778665, "95.00 percentile latency (ns)": 5394035, "97.00 percentile latency (ns)": 6858773, "99.00 percentile latency (ns)": 9602024, "99.90 percentile latency (ns)": 39363655, "Max latency (ns)": 79693809, "Mean latency (ns)": 3712467, "Min duration satisfied": "Yes", "Min latency (ns)": 2913794, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 268.98, "QPS w/o loadgen overhead": 269.36, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.444, "characteristics.good": 29722, "characteristics.power": 20.288800000000002, "characteristics.power.normalized_per_core": 0.16907333333333335, "characteristics.power.normalized_per_processor": 20.288800000000002, "characteristics.samples_per_second": 269.36, "characteristics.samples_per_second.normalized_per_core": 2.244666666666667, "characteristics.samples_per_second.normalized_per_processor": 269.36, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c8bc634507f8b4e8", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4340208, "90.00 percentile latency (ns)": 4831943, "90th percentile latency (ns)": 4831943, "95.00 percentile latency (ns)": 6739688, "97.00 percentile latency (ns)": 8492073, "99.00 percentile latency (ns)": 11164128, "99.90 percentile latency (ns)": 28509981, "Max latency (ns)": 75223883, "Mean latency (ns)": 4719617, "Min duration satisfied": "Yes", "Min latency (ns)": 3946038, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 211.58, "QPS w/o loadgen overhead": 211.88, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.456, "characteristics.good": 30728, "characteristics.power": 20.191849999999963, "characteristics.power.normalized_per_core": 0.16826541666666636, "characteristics.power.normalized_per_processor": 20.191849999999963, "characteristics.samples_per_second": 211.88, "characteristics.samples_per_second.normalized_per_core": 1.7656666666666667, "characteristics.samples_per_second.normalized_per_processor": 211.88, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "b1aa8de4c1ba6782", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5642449, "90.00 percentile latency (ns)": 6821245, "90th percentile latency (ns)": 6821245, "95.00 percentile latency (ns)": 8666315, "97.00 percentile latency (ns)": 10812701, "99.00 percentile latency (ns)": 13665247, "99.90 percentile latency (ns)": 63989948, "Max latency (ns)": 111954722, "Mean latency (ns)": 6201014, "Min duration satisfied": "Yes", "Min latency (ns)": 4956459, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 161.04, "QPS w/o loadgen overhead": 161.26, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.528, "characteristics.good": 29264, "characteristics.power": 19.9696833333333, "characteristics.power.normalized_per_core": 0.1664140277777775, "characteristics.power.normalized_per_processor": 19.9696833333333, "characteristics.samples_per_second": 161.26, "characteristics.samples_per_second.normalized_per_core": 1.3438333333333332, "characteristics.samples_per_second.normalized_per_processor": 161.26, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "861e1c7ccda04d2d", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4098168, "90.00 percentile latency (ns)": 6067120, "90th percentile latency (ns)": 6067120, "95.00 percentile latency (ns)": 7576725, "97.00 percentile latency (ns)": 8756522, "99.00 percentile latency (ns)": 11505188, "99.90 percentile latency (ns)": 48980883, "Max latency (ns)": 136063475, "Mean latency (ns)": 4746816, "Min duration satisfied": "Yes", "Min latency (ns)": 3586722, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 209.88, "QPS w/o loadgen overhead": 210.67, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.036, "characteristics.good": 33018, "characteristics.power": 24.546833333333343, "characteristics.power.normalized_per_core": 0.20455694444444453, "characteristics.power.normalized_per_processor": 24.546833333333343, "characteristics.samples_per_second": 210.67, "characteristics.samples_per_second.normalized_per_core": 1.7555833333333333, "characteristics.samples_per_second.normalized_per_processor": 210.67, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "48e28dd1de7a4e61", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 16788660, "90.00 percentile latency (ns)": 21488871, "90th percentile latency (ns)": 21488871, "95.00 percentile latency (ns)": 23942219, "97.00 percentile latency (ns)": 24711608, "99.00 percentile latency (ns)": 33368065, "99.90 percentile latency (ns)": 83772813, "Max latency (ns)": 129348687, "Mean latency (ns)": 17594625, "Min duration satisfied": "Yes", "Min latency (ns)": 14104111, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 56.77, "QPS w/o loadgen overhead": 56.84, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.642, "characteristics.good": 38321, "characteristics.power": 26.717166666666696, "characteristics.power.normalized_per_core": 0.2226430555555558, "characteristics.power.normalized_per_processor": 26.717166666666696, "characteristics.samples_per_second": 56.84, "characteristics.samples_per_second.normalized_per_core": 0.4736666666666667, "characteristics.samples_per_second.normalized_per_processor": 56.84, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite1-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "99a57dc176834c96", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10068923, "90.00 percentile latency (ns)": 16896820, "90th percentile latency (ns)": 16896820, "95.00 percentile latency (ns)": 18957921, "97.00 percentile latency (ns)": 20346252, "99.00 percentile latency (ns)": 26969356, "99.90 percentile latency (ns)": 91451293, "Max latency (ns)": 132020057, "Mean latency (ns)": 11687638, "Min duration satisfied": "Yes", "Min latency (ns)": 8694860, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 85.48, "QPS w/o loadgen overhead": 85.56, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.604, "characteristics.good": 34802, "characteristics.power": 20.26066666666667, "characteristics.power.normalized_per_core": 0.1688388888888889, "characteristics.power.normalized_per_processor": 20.26066666666667, "characteristics.samples_per_second": 85.56, "characteristics.samples_per_second.normalized_per_core": 0.713, "characteristics.samples_per_second.normalized_per_processor": 85.56, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ca118b4990a204e1", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9077702, "90.00 percentile latency (ns)": 11225873, "90th percentile latency (ns)": 11225873, "95.00 percentile latency (ns)": 13691482, "97.00 percentile latency (ns)": 15085603, "99.00 percentile latency (ns)": 22884901, "99.90 percentile latency (ns)": 69236771, "Max latency (ns)": 97011245, "Mean latency (ns)": 9514962, "Min duration satisfied": "Yes", "Min latency (ns)": 7612679, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 104.87, "QPS w/o loadgen overhead": 105.1, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.262, "characteristics.good": 34631, "characteristics.power": 25.440849999999998, "characteristics.power.normalized_per_core": 0.21200708333333332, "characteristics.power.normalized_per_processor": 25.440849999999998, "characteristics.samples_per_second": 105.1, "characteristics.samples_per_second.normalized_per_core": 0.8758333333333332, "characteristics.samples_per_second.normalized_per_processor": 105.1, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "aae81f88e0a39599", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11130326, "90.00 percentile latency (ns)": 17121202, "90th percentile latency (ns)": 17121202, "95.00 percentile latency (ns)": 18429414, "97.00 percentile latency (ns)": 19261134, "99.00 percentile latency (ns)": 28785963, "99.90 percentile latency (ns)": 91108410, "Max latency (ns)": 122111849, "Mean latency (ns)": 12573505, "Min duration satisfied": "Yes", "Min latency (ns)": 9572222, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 79.46, "QPS w/o loadgen overhead": 79.53, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.734, "characteristics.good": 34367, "characteristics.power": 20.405783333333325, "characteristics.power.normalized_per_core": 0.17004819444444438, "characteristics.power.normalized_per_processor": 20.405783333333325, "characteristics.samples_per_second": 79.53, "characteristics.samples_per_second.normalized_per_core": 0.6627500000000001, "characteristics.samples_per_second.normalized_per_processor": 79.53, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "f6d9044bef42c687", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3643170, "90.00 percentile latency (ns)": 4146295, "90th percentile latency (ns)": 4146295, "95.00 percentile latency (ns)": 4849714, "97.00 percentile latency (ns)": 5695161, "99.00 percentile latency (ns)": 8902408, "99.90 percentile latency (ns)": 41693477, "Max latency (ns)": 111844487, "Mean latency (ns)": 3948045, "Min duration satisfied": "Yes", "Min latency (ns)": 3175085, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 252.23, "QPS w/o loadgen overhead": 253.29, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.33, "characteristics.good": 31665, "characteristics.power": 23.10059999999997, "characteristics.power.normalized_per_core": 0.19250499999999976, "characteristics.power.normalized_per_processor": 23.10059999999997, "characteristics.samples_per_second": 253.29, "characteristics.samples_per_second.normalized_per_core": 2.11075, "characteristics.samples_per_second.normalized_per_processor": 253.29, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4160c3cdee460987", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 1961159, "90.00 percentile latency (ns)": 2230709, "90th percentile latency (ns)": 2230709, "95.00 percentile latency (ns)": 2512715, "97.00 percentile latency (ns)": 3075935, "99.00 percentile latency (ns)": 4424100, "99.90 percentile latency (ns)": 25260456, "Max latency (ns)": 79646906, "Mean latency (ns)": 2142206, "Min duration satisfied": "Yes", "Min latency (ns)": 1749051, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 465.05, "QPS w/o loadgen overhead": 466.81, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 57.918, "characteristics.good": 28959, "characteristics.power": 24.597616666666692, "characteristics.power.normalized_per_core": 0.2049801388888891, "characteristics.power.normalized_per_processor": 24.597616666666692, "characteristics.samples_per_second": 466.81, "characteristics.samples_per_second.normalized_per_core": 3.8900833333333336, "characteristics.samples_per_second.normalized_per_processor": 466.81, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 500, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "7069f34ebc06808a", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 12181186, "90.00 percentile latency (ns)": 15314498, "90th percentile latency (ns)": 15314498, "95.00 percentile latency (ns)": 17713044, "97.00 percentile latency (ns)": 18957131, "99.00 percentile latency (ns)": 26293749, "99.90 percentile latency (ns)": 66273194, "Max latency (ns)": 86073327, "Mean latency (ns)": 13343739, "Min duration satisfied": "Yes", "Min latency (ns)": 11153917, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 74.83, "QPS w/o loadgen overhead": 74.94, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.676, "characteristics.good": 35838, "characteristics.power": 26.47369999999999, "characteristics.power.normalized_per_core": 0.22061416666666658, "characteristics.power.normalized_per_processor": 26.47369999999999, "characteristics.samples_per_second": 74.94, "characteristics.samples_per_second.normalized_per_core": 0.6244999999999999, "characteristics.samples_per_second.normalized_per_processor": 74.94, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ba48ca499baf2933", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 1938673, "90.00 percentile latency (ns)": 2145877, "90th percentile latency (ns)": 2145877, "95.00 percentile latency (ns)": 2668495, "97.00 percentile latency (ns)": 3316480, "99.00 percentile latency (ns)": 4958059, "99.90 percentile latency (ns)": 26086114, "Max latency (ns)": 98213600, "Mean latency (ns)": 2127361, "Min duration satisfied": "Yes", "Min latency (ns)": 1684550, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 469.29, "QPS w/o loadgen overhead": 470.07, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 43.468, "characteristics.good": 21734, "characteristics.power": 19.76489999999999, "characteristics.power.normalized_per_core": 0.16470749999999992, "characteristics.power.normalized_per_processor": 19.76489999999999, "characteristics.samples_per_second": 470.07, "characteristics.samples_per_second.normalized_per_core": 3.91725, "characteristics.samples_per_second.normalized_per_processor": 470.07, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "942108ca9071871a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 52390715, "90.00 percentile latency (ns)": 67754253, "90th percentile latency (ns)": 67754253, "95.00 percentile latency (ns)": 70348269, "97.00 percentile latency (ns)": 73303283, "99.00 percentile latency (ns)": 82868245, "99.90 percentile latency (ns)": 135484751, "Max latency (ns)": 168887474, "Mean latency (ns)": 56749943, "Min duration satisfied": "Yes", "Min latency (ns)": 48470052, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.62, "QPS w/o loadgen overhead": 17.62, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 80.676, "characteristics.good": 40338, "characteristics.power": 28.110016638935118, "characteristics.power.normalized_per_core": 0.23425013865779265, "characteristics.power.normalized_per_processor": 28.110016638935118, "characteristics.samples_per_second": 17.62, "characteristics.samples_per_second.normalized_per_core": 0.14683333333333334, "characteristics.samples_per_second.normalized_per_processor": 17.62, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite4-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "7e8ca043d49f983c", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10116552, "90.00 percentile latency (ns)": 16051294, "90th percentile latency (ns)": 16051294, "95.00 percentile latency (ns)": 17369116, "97.00 percentile latency (ns)": 18169298, "99.00 percentile latency (ns)": 26397666, "99.90 percentile latency (ns)": 79916187, "Max latency (ns)": 121147232, "Mean latency (ns)": 11469463, "Min duration satisfied": "Yes", "Min latency (ns)": 8864854, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 87.11, "QPS w/o loadgen overhead": 87.19, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.43, "characteristics.good": 35215, "characteristics.power": 20.223566666666652, "characteristics.power.normalized_per_core": 0.1685297222222221, "characteristics.power.normalized_per_processor": 20.223566666666652, "characteristics.samples_per_second": 87.19, "characteristics.samples_per_second.normalized_per_core": 0.7265833333333334, "characteristics.samples_per_second.normalized_per_processor": 87.19, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "0942c2a4044c7965", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8010390, "90.00 percentile latency (ns)": 10347189, "90th percentile latency (ns)": 10347189, "95.00 percentile latency (ns)": 14184363, "97.00 percentile latency (ns)": 15420046, "99.00 percentile latency (ns)": 19839660, "99.90 percentile latency (ns)": 74003003, "Max latency (ns)": 137014167, "Mean latency (ns)": 8904344, "Min duration satisfied": "Yes", "Min latency (ns)": 7315348, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 112.19, "QPS w/o loadgen overhead": 112.3, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.474, "characteristics.good": 33737, "characteristics.power": 20.186299999999992, "characteristics.power.normalized_per_core": 0.16821916666666661, "characteristics.power.normalized_per_processor": 20.186299999999992, "characteristics.samples_per_second": 112.3, "characteristics.samples_per_second.normalized_per_core": 0.9358333333333333, "characteristics.samples_per_second.normalized_per_processor": 112.3, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c6242963e497db6c", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8205284, "90.00 percentile latency (ns)": 9930068, "90th percentile latency (ns)": 9930068, "95.00 percentile latency (ns)": 12211031, "97.00 percentile latency (ns)": 14017437, "99.00 percentile latency (ns)": 20770704, "99.90 percentile latency (ns)": 66929746, "Max latency (ns)": 81404605, "Mean latency (ns)": 8897915, "Min duration satisfied": "Yes", "Min latency (ns)": 7318556, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 112.27, "QPS w/o loadgen overhead": 112.39, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.24, "characteristics.good": 33620, "characteristics.power": 20.20101666666665, "characteristics.power.normalized_per_core": 0.16834180555555542, "characteristics.power.normalized_per_processor": 20.20101666666665, "characteristics.samples_per_second": 112.39, "characteristics.samples_per_second.normalized_per_core": 0.9365833333333333, "characteristics.samples_per_second.normalized_per_processor": 112.39, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "cea8201e82beab76", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4970164, "90.00 percentile latency (ns)": 6809054, "90th percentile latency (ns)": 6809054, "95.00 percentile latency (ns)": 8479048, "97.00 percentile latency (ns)": 9828545, "99.00 percentile latency (ns)": 12153351, "99.90 percentile latency (ns)": 30156229, "Max latency (ns)": 87896476, "Mean latency (ns)": 5525220, "Min duration satisfied": "Yes", "Min latency (ns)": 4416349, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 180.42, "QPS w/o loadgen overhead": 180.99, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.348, "characteristics.good": 32174, "characteristics.power": 24.921233333333323, "characteristics.power.normalized_per_core": 0.20767694444444434, "characteristics.power.normalized_per_processor": 24.921233333333323, "characteristics.samples_per_second": 180.99, "characteristics.samples_per_second.normalized_per_core": 1.50825, "characteristics.samples_per_second.normalized_per_processor": 180.99, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "fcf90bb0abad7b69", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3898617, "90.00 percentile latency (ns)": 5422324, "90th percentile latency (ns)": 5422324, "95.00 percentile latency (ns)": 6852940, "97.00 percentile latency (ns)": 7960887, "99.00 percentile latency (ns)": 10340725, "99.90 percentile latency (ns)": 28495932, "Max latency (ns)": 74529157, "Mean latency (ns)": 4374770, "Min duration satisfied": "Yes", "Min latency (ns)": 3441063, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 227.57, "QPS w/o loadgen overhead": 228.58, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.246, "characteristics.good": 31623, "characteristics.power": 24.635066666666663, "characteristics.power.normalized_per_core": 0.20529222222222218, "characteristics.power.normalized_per_processor": 24.635066666666663, "characteristics.samples_per_second": 228.58, "characteristics.samples_per_second.normalized_per_core": 1.9048333333333334, "characteristics.samples_per_second.normalized_per_processor": 228.58, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "eb8e78af3ddcac92", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3294329, "90.00 percentile latency (ns)": 4155685, "90th percentile latency (ns)": 4155685, "95.00 percentile latency (ns)": 5188377, "97.00 percentile latency (ns)": 6214183, "99.00 percentile latency (ns)": 9453212, "99.90 percentile latency (ns)": 36187142, "Max latency (ns)": 130500545, "Mean latency (ns)": 3673477, "Min duration satisfied": "Yes", "Min latency (ns)": 2833923, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 271.4, "QPS w/o loadgen overhead": 272.22, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.668, "characteristics.good": 30834, "characteristics.power": 23.64273333333333, "characteristics.power.normalized_per_core": 0.19702277777777774, "characteristics.power.normalized_per_processor": 23.64273333333333, "characteristics.samples_per_second": 272.22, "characteristics.samples_per_second.normalized_per_core": 2.2685000000000004, "characteristics.samples_per_second.normalized_per_processor": 272.22, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "de3e99b4d1dab523", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4816120, "90.00 percentile latency (ns)": 6940885, "90th percentile latency (ns)": 6940885, "95.00 percentile latency (ns)": 8608322, "97.00 percentile latency (ns)": 9725812, "99.00 percentile latency (ns)": 12230500, "99.90 percentile latency (ns)": 55170538, "Max latency (ns)": 75521201, "Mean latency (ns)": 5497462, "Min duration satisfied": "Yes", "Min latency (ns)": 4333634, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 181.27, "QPS w/o loadgen overhead": 181.9, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.384, "characteristics.good": 33192, "characteristics.power": 25.323900000000002, "characteristics.power.normalized_per_core": 0.2110325, "characteristics.power.normalized_per_processor": 25.323900000000002, "characteristics.samples_per_second": 181.9, "characteristics.samples_per_second.normalized_per_core": 1.5158333333333334, "characteristics.samples_per_second.normalized_per_processor": 181.9, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "73234957b573091f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2476397, "90.00 percentile latency (ns)": 3066855, "90th percentile latency (ns)": 3066855, "95.00 percentile latency (ns)": 4063474, "97.00 percentile latency (ns)": 4811541, "99.00 percentile latency (ns)": 7659186, "99.90 percentile latency (ns)": 30138862, "Max latency (ns)": 106838091, "Mean latency (ns)": 2788294, "Min duration satisfied": "Yes", "Min latency (ns)": 2187072, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 357.98, "QPS w/o loadgen overhead": 358.64, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.11, "characteristics.good": 24555, "characteristics.power": 19.669166666666676, "characteristics.power.normalized_per_core": 0.1639097222222223, "characteristics.power.normalized_per_processor": 19.669166666666676, "characteristics.samples_per_second": 358.64, "characteristics.samples_per_second.normalized_per_core": 2.9886666666666666, "characteristics.samples_per_second.normalized_per_processor": 358.64, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "3eb84eaaf4429bd7", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4910176, "90.00 percentile latency (ns)": 5958194, "90th percentile latency (ns)": 5958194, "95.00 percentile latency (ns)": 7338965, "97.00 percentile latency (ns)": 9296754, "99.00 percentile latency (ns)": 12669668, "99.90 percentile latency (ns)": 56919452, "Max latency (ns)": 122619405, "Mean latency (ns)": 5429563, "Min duration satisfied": "Yes", "Min latency (ns)": 4342984, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 183.91, "QPS w/o loadgen overhead": 184.18, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 62.07, "characteristics.good": 31035, "characteristics.power": 19.919716666666645, "characteristics.power.normalized_per_core": 0.16599763888888872, "characteristics.power.normalized_per_processor": 19.919716666666645, "characteristics.samples_per_second": 184.18, "characteristics.samples_per_second.normalized_per_core": 1.5348333333333335, "characteristics.samples_per_second.normalized_per_processor": 184.18, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d102384370993c20", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3212719, "90.00 percentile latency (ns)": 5168166, "90th percentile latency (ns)": 5168166, "95.00 percentile latency (ns)": 6475809, "97.00 percentile latency (ns)": 7534417, "99.00 percentile latency (ns)": 10119203, "99.90 percentile latency (ns)": 34414402, "Max latency (ns)": 123172729, "Mean latency (ns)": 3803621, "Min duration satisfied": "Yes", "Min latency (ns)": 2660439, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 261.73, "QPS w/o loadgen overhead": 262.91, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.224, "characteristics.good": 30612, "characteristics.power": 24.088183333333326, "characteristics.power.normalized_per_core": 0.20073486111111105, "characteristics.power.normalized_per_processor": 24.088183333333326, "characteristics.samples_per_second": 262.91, "characteristics.samples_per_second.normalized_per_core": 2.190916666666667, "characteristics.samples_per_second.normalized_per_processor": 262.91, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "7948e99da2434042", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 1612615, "90.00 percentile latency (ns)": 1802060, "90th percentile latency (ns)": 1802060, "95.00 percentile latency (ns)": 2370600, "97.00 percentile latency (ns)": 2789019, "99.00 percentile latency (ns)": 4093457, "99.90 percentile latency (ns)": 22684358, "Max latency (ns)": 75931763, "Mean latency (ns)": 1768185, "Min duration satisfied": "Yes", "Min latency (ns)": 1273560, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 564.44, "QPS w/o loadgen overhead": 565.55, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 42.384, "characteristics.good": 21192, "characteristics.power": 19.307833333333313, "characteristics.power.normalized_per_core": 0.16089861111111095, "characteristics.power.normalized_per_processor": 19.307833333333313, "characteristics.samples_per_second": 565.55, "characteristics.samples_per_second.normalized_per_core": 4.712916666666667, "characteristics.samples_per_second.normalized_per_processor": 565.55, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 500, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "604d8c92a6b98cc8", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 13330121, "90.00 percentile latency (ns)": 19891031, "90th percentile latency (ns)": 19891031, "95.00 percentile latency (ns)": 20889930, "97.00 percentile latency (ns)": 21914047, "99.00 percentile latency (ns)": 32638019, "99.90 percentile latency (ns)": 111747019, "Max latency (ns)": 136531740, "Mean latency (ns)": 15456335, "Min duration satisfied": "Yes", "Min latency (ns)": 12122155, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 64.61, "QPS w/o loadgen overhead": 64.7, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.012, "characteristics.good": 37506, "characteristics.power": 26.194366666666653, "characteristics.power.normalized_per_core": 0.21828638888888877, "characteristics.power.normalized_per_processor": 26.194366666666653, "characteristics.samples_per_second": 64.7, "characteristics.samples_per_second.normalized_per_core": 0.5391666666666667, "characteristics.samples_per_second.normalized_per_processor": 64.7, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.3-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.3-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.3_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c4cd5277452681cd", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4850903, "90.00 percentile latency (ns)": 5897061, "90th percentile latency (ns)": 5897061, "95.00 percentile latency (ns)": 7375624, "97.00 percentile latency (ns)": 8889925, "99.00 percentile latency (ns)": 12105911, "99.90 percentile latency (ns)": 43993932, "Max latency (ns)": 112061141, "Mean latency (ns)": 5344462, "Min duration satisfied": "Yes", "Min latency (ns)": 4349247, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 186.8, "QPS w/o loadgen overhead": 187.11, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.486, "characteristics.good": 32243, "characteristics.power": 20.90964999999997, "characteristics.power.normalized_per_core": 0.17424708333333308, "characteristics.power.normalized_per_processor": 20.90964999999997, "characteristics.samples_per_second": 187.11, "characteristics.samples_per_second.normalized_per_core": 1.55925, "characteristics.samples_per_second.normalized_per_processor": 187.11, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-small_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "e8a8e432e9684a09", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20342460, "90.00 percentile latency (ns)": 27726474, "90th percentile latency (ns)": 27726474, "95.00 percentile latency (ns)": 29065095, "97.00 percentile latency (ns)": 30234588, "99.00 percentile latency (ns)": 39332282, "99.90 percentile latency (ns)": 86504020, "Max latency (ns)": 118384094, "Mean latency (ns)": 22009131, "Min duration satisfied": "Yes", "Min latency (ns)": 19306924, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 45.39, "QPS w/o loadgen overhead": 45.44, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 77.542, "characteristics.good": 38771, "characteristics.power": 27.40504999999996, "characteristics.power.normalized_per_core": 0.22837541666666633, "characteristics.power.normalized_per_processor": 27.40504999999996, "characteristics.samples_per_second": 45.44, "characteristics.samples_per_second.normalized_per_core": 0.37866666666666665, "characteristics.samples_per_second.normalized_per_processor": 45.44, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite2-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "76744db9bd4b1086", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2606485, "90.00 percentile latency (ns)": 2818686, "90th percentile latency (ns)": 2818686, "95.00 percentile latency (ns)": 3019527, "97.00 percentile latency (ns)": 3919727, "99.00 percentile latency (ns)": 6331067, "99.90 percentile latency (ns)": 22142716, "Max latency (ns)": 71836983, "Mean latency (ns)": 2775435, "Min duration satisfied": "Yes", "Min latency (ns)": 2334537, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 359.72, "QPS w/o loadgen overhead": 360.3, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.75, "characteristics.good": 28375, "characteristics.power": 19.6376333333333, "characteristics.power.normalized_per_core": 0.1636469444444442, "characteristics.power.normalized_per_processor": 19.6376333333333, "characteristics.samples_per_second": 360.3, "characteristics.samples_per_second.normalized_per_core": 3.0025, "characteristics.samples_per_second.normalized_per_processor": 360.3, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.5-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "983963835d501d73", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3443321, "90.00 percentile latency (ns)": 4885464, "90th percentile latency (ns)": 4885464, "95.00 percentile latency (ns)": 6511652, "97.00 percentile latency (ns)": 7859807, "99.00 percentile latency (ns)": 10664085, "99.90 percentile latency (ns)": 36329778, "Max latency (ns)": 117415463, "Mean latency (ns)": 3965685, "Min duration satisfied": "Yes", "Min latency (ns)": 2999878, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 251.76, "QPS w/o loadgen overhead": 252.16, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 57.002, "characteristics.good": 28501, "characteristics.power": 19.830933333333352, "characteristics.power.normalized_per_core": 0.16525777777777792, "characteristics.power.normalized_per_processor": 19.830933333333352, "characteristics.samples_per_second": 252.16, "characteristics.samples_per_second.normalized_per_core": 2.1013333333333333, "characteristics.samples_per_second.normalized_per_processor": 252.16, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "b8717d949fdb10fa", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 15796054, "90.00 percentile latency (ns)": 21690501, "90th percentile latency (ns)": 21690501, "95.00 percentile latency (ns)": 23132942, "97.00 percentile latency (ns)": 24169136, "99.00 percentile latency (ns)": 34124798, "99.90 percentile latency (ns)": 127934583, "Max latency (ns)": 154512302, "Mean latency (ns)": 17285338, "Min duration satisfied": "Yes", "Min latency (ns)": 13343500, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 57.78, "QPS w/o loadgen overhead": 57.85, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.69, "characteristics.good": 37845, "characteristics.power": 26.399749999999987, "characteristics.power.normalized_per_core": 0.21999791666666654, "characteristics.power.normalized_per_processor": 26.399749999999987, "characteristics.samples_per_second": 57.85, "characteristics.samples_per_second.normalized_per_core": 0.48208333333333336, "characteristics.samples_per_second.normalized_per_processor": 57.85, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.4-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.4-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.4_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6691c8bac8c21103", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 1793200, "90.00 percentile latency (ns)": 2078708, "90th percentile latency (ns)": 2078708, "95.00 percentile latency (ns)": 2604874, "97.00 percentile latency (ns)": 3208292, "99.00 percentile latency (ns)": 5023074, "99.90 percentile latency (ns)": 25801481, "Max latency (ns)": 99697130, "Mean latency (ns)": 1999008, "Min duration satisfied": "Yes", "Min latency (ns)": 1538143, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 498.19, "QPS w/o loadgen overhead": 500.25, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 52.446, "characteristics.good": 26223, "characteristics.power": 22.908683333333347, "characteristics.power.normalized_per_core": 0.19090569444444455, "characteristics.power.normalized_per_processor": 22.908683333333347, "characteristics.samples_per_second": 500.25, "characteristics.samples_per_second.normalized_per_core": 4.16875, "characteristics.samples_per_second.normalized_per_processor": 500.25, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "b72d2cb4b798eabd", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7663810, "90.00 percentile latency (ns)": 9368466, "90th percentile latency (ns)": 9368466, "95.00 percentile latency (ns)": 11834601, "97.00 percentile latency (ns)": 13890645, "99.00 percentile latency (ns)": 19110465, "99.90 percentile latency (ns)": 79740440, "Max latency (ns)": 125161946, "Mean latency (ns)": 8408980, "Min duration satisfied": "Yes", "Min latency (ns)": 7076032, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 118.68, "QPS w/o loadgen overhead": 118.92, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.724, "characteristics.good": 36862, "characteristics.power": 24.891149999999975, "characteristics.power.normalized_per_core": 0.20742624999999978, "characteristics.power.normalized_per_processor": 24.891149999999975, "characteristics.samples_per_second": 118.92, "characteristics.samples_per_second.normalized_per_core": 0.991, "characteristics.samples_per_second.normalized_per_processor": 118.92, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-large_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "b41db9a3e87679fe", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4401970, "90.00 percentile latency (ns)": 5476482, "90th percentile latency (ns)": 5476482, "95.00 percentile latency (ns)": 6682179, "97.00 percentile latency (ns)": 7851161, "99.00 percentile latency (ns)": 10922436, "99.90 percentile latency (ns)": 54054283, "Max latency (ns)": 98608219, "Mean latency (ns)": 4863622, "Min duration satisfied": "Yes", "Min latency (ns)": 4018173, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 204.67, "QPS w/o loadgen overhead": 205.61, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.604, "characteristics.good": 33302, "characteristics.power": 26.90060000000003, "characteristics.power.normalized_per_core": 0.2241716666666669, "characteristics.power.normalized_per_processor": 26.90060000000003, "characteristics.samples_per_second": 205.61, "characteristics.samples_per_second.normalized_per_core": 1.7134166666666668, "characteristics.samples_per_second.normalized_per_processor": 205.61, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "722b4b18aa6ed033", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6883796, "90.00 percentile latency (ns)": 8377014, "90th percentile latency (ns)": 8377014, "95.00 percentile latency (ns)": 10224424, "97.00 percentile latency (ns)": 12021752, "99.00 percentile latency (ns)": 16800749, "99.90 percentile latency (ns)": 65809241, "Max latency (ns)": 85384163, "Mean latency (ns)": 7544478, "Min duration satisfied": "Yes", "Min latency (ns)": 6297145, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 132.24, "QPS w/o loadgen overhead": 132.55, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.106, "characteristics.good": 34553, "characteristics.power": 26.116683333333324, "characteristics.power.normalized_per_core": 0.2176390277777777, "characteristics.power.normalized_per_processor": 26.116683333333324, "characteristics.samples_per_second": 132.55, "characteristics.samples_per_second.normalized_per_core": 1.1045833333333335, "characteristics.samples_per_second.normalized_per_processor": 132.55, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4735ac2939a19fa7", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2700409, "90.00 percentile latency (ns)": 3094475, "90th percentile latency (ns)": 3094475, "95.00 percentile latency (ns)": 3756230, "97.00 percentile latency (ns)": 4688530, "99.00 percentile latency (ns)": 7412361, "99.90 percentile latency (ns)": 31629666, "Max latency (ns)": 129703256, "Mean latency (ns)": 2986152, "Min duration satisfied": "Yes", "Min latency (ns)": 2375146, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 333.41, "QPS w/o loadgen overhead": 334.88, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.822, "characteristics.good": 29911, "characteristics.power": 23.26799999999999, "characteristics.power.normalized_per_core": 0.1938999999999999, "characteristics.power.normalized_per_processor": 23.26799999999999, "characteristics.samples_per_second": 334.88, "characteristics.samples_per_second.normalized_per_core": 2.7906666666666666, "characteristics.samples_per_second.normalized_per_processor": 334.88, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "94d09be8382e919f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4619376, "90.00 percentile latency (ns)": 5305251, "90th percentile latency (ns)": 5305251, "95.00 percentile latency (ns)": 6239929, "97.00 percentile latency (ns)": 7514482, "99.00 percentile latency (ns)": 11782674, "99.90 percentile latency (ns)": 52385660, "Max latency (ns)": 114098887, "Mean latency (ns)": 5022017, "Min duration satisfied": "Yes", "Min latency (ns)": 4153645, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 198.35, "QPS w/o loadgen overhead": 199.12, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.336, "characteristics.good": 34168, "characteristics.power": 23.78865, "characteristics.power.normalized_per_core": 0.19823875, "characteristics.power.normalized_per_processor": 23.78865, "characteristics.samples_per_second": 199.12, "characteristics.samples_per_second.normalized_per_core": 1.6593333333333333, "characteristics.samples_per_second.normalized_per_processor": 199.12, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-small_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "fb5fc0cb28262356", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6408609, "90.00 percentile latency (ns)": 10233899, "90th percentile latency (ns)": 10233899, "95.00 percentile latency (ns)": 11816274, "97.00 percentile latency (ns)": 12850753, "99.00 percentile latency (ns)": 16243416, "99.90 percentile latency (ns)": 75792104, "Max latency (ns)": 131115531, "Mean latency (ns)": 7547977, "Min duration satisfied": "Yes", "Min latency (ns)": 5532409, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 132.12, "QPS w/o loadgen overhead": 132.49, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.834, "characteristics.good": 34417, "characteristics.power": 24.39201666666669, "characteristics.power.normalized_per_core": 0.20326680555555576, "characteristics.power.normalized_per_processor": 24.39201666666669, "characteristics.samples_per_second": 132.49, "characteristics.samples_per_second.normalized_per_core": 1.1040833333333333, "characteristics.samples_per_second.normalized_per_processor": 132.49, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "19ac49ab8c7b32da", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 13813975, "90.00 percentile latency (ns)": 20706785, "90th percentile latency (ns)": 20706785, "95.00 percentile latency (ns)": 21374353, "97.00 percentile latency (ns)": 21940975, "99.00 percentile latency (ns)": 30072408, "99.90 percentile latency (ns)": 76406162, "Max latency (ns)": 105469132, "Mean latency (ns)": 15895506, "Min duration satisfied": "Yes", "Min latency (ns)": 12737234, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 62.86, "QPS w/o loadgen overhead": 62.91, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.19, "characteristics.good": 35595, "characteristics.power": 20.938133333333344, "characteristics.power.normalized_per_core": 0.17448444444444453, "characteristics.power.normalized_per_processor": 20.938133333333344, "characteristics.samples_per_second": 62.91, "characteristics.samples_per_second.normalized_per_core": 0.52425, "characteristics.samples_per_second.normalized_per_processor": 62.91, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite0-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "a25fc44b682bd678", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 31470866, "90.00 percentile latency (ns)": 42720272, "90th percentile latency (ns)": 42720272, "95.00 percentile latency (ns)": 46836714, "97.00 percentile latency (ns)": 48511990, "99.00 percentile latency (ns)": 56649510, "99.90 percentile latency (ns)": 102651561, "Max latency (ns)": 147487515, "Mean latency (ns)": 34598096, "Min duration satisfied": "Yes", "Min latency (ns)": 28584271, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 28.89, "QPS w/o loadgen overhead": 28.9, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 79.268, "characteristics.good": 39634, "characteristics.power": 27.695983333333334, "characteristics.power.normalized_per_core": 0.23079986111111112, "characteristics.power.normalized_per_processor": 27.695983333333334, "characteristics.samples_per_second": 28.9, "characteristics.samples_per_second.normalized_per_core": 0.24083333333333332, "characteristics.samples_per_second.normalized_per_processor": 28.9, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite3-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6aa9e79d604b6e99", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4190750, "90.00 percentile latency (ns)": 4911710, "90th percentile latency (ns)": 4911710, "95.00 percentile latency (ns)": 5879947, "97.00 percentile latency (ns)": 7038303, "99.00 percentile latency (ns)": 10796201, "99.90 percentile latency (ns)": 48957002, "Max latency (ns)": 115493842, "Mean latency (ns)": 4571596, "Min duration satisfied": "Yes", "Min latency (ns)": 3687015, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 217.88, "QPS w/o loadgen overhead": 218.74, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.216, "characteristics.good": 33108, "characteristics.power": 23.25205000000002, "characteristics.power.normalized_per_core": 0.19376708333333348, "characteristics.power.normalized_per_processor": 23.25205000000002, "characteristics.samples_per_second": 218.74, "characteristics.samples_per_second.normalized_per_core": 1.8228333333333333, "characteristics.samples_per_second.normalized_per_processor": 218.74, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-small_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "2eaf95e631204c16", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 43414374, "90.00 percentile latency (ns)": 54639225, "90th percentile latency (ns)": 54639225, "95.00 percentile latency (ns)": 57335837, "97.00 percentile latency (ns)": 59572022, "99.00 percentile latency (ns)": 69197931, "99.90 percentile latency (ns)": 114044467, "Max latency (ns)": 180294879, "Mean latency (ns)": 46167418, "Min duration satisfied": "Yes", "Min latency (ns)": 39070500, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 21.65, "QPS w/o loadgen overhead": 21.66, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.344, "characteristics.good": 38172, "characteristics.power": 21.216900000000003, "characteristics.power.normalized_per_core": 0.17680750000000003, "characteristics.power.normalized_per_processor": 21.216900000000003, "characteristics.samples_per_second": 21.66, "characteristics.samples_per_second.normalized_per_core": 0.1805, "characteristics.samples_per_second.normalized_per_processor": 21.66, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "efficientnet-lite3-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4696adf3e0fb9328", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3613564, "90.00 percentile latency (ns)": 4697962, "90th percentile latency (ns)": 4697962, "95.00 percentile latency (ns)": 5982688, "97.00 percentile latency (ns)": 7432318, "99.00 percentile latency (ns)": 10462111, "99.90 percentile latency (ns)": 41091962, "Max latency (ns)": 118523831, "Mean latency (ns)": 4072582, "Min duration satisfied": "Yes", "Min latency (ns)": 3177287, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 245.08, "QPS w/o loadgen overhead": 245.54, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 54.568, "characteristics.good": 27284, "characteristics.power": 19.84628333333334, "characteristics.power.normalized_per_core": 0.16538569444444448, "characteristics.power.normalized_per_processor": 19.84628333333334, "characteristics.samples_per_second": 245.54, "characteristics.samples_per_second.normalized_per_core": 2.0461666666666667, "characteristics.samples_per_second.normalized_per_processor": 245.54, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "be11b6d06b747648", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2457198, "90.00 percentile latency (ns)": 2711065, "90th percentile latency (ns)": 2711065, "95.00 percentile latency (ns)": 3340437, "97.00 percentile latency (ns)": 3802250, "99.00 percentile latency (ns)": 5780354, "99.90 percentile latency (ns)": 33881325, "Max latency (ns)": 77816860, "Mean latency (ns)": 2671392, "Min duration satisfied": "Yes", "Min latency (ns)": 2100317, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 373.46, "QPS w/o loadgen overhead": 374.34, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.478, "characteristics.good": 24739, "characteristics.power": 22.87413333333333, "characteristics.power.normalized_per_core": 0.19061777777777775, "characteristics.power.normalized_per_processor": 22.87413333333333, "characteristics.samples_per_second": 374.34, "characteristics.samples_per_second.normalized_per_core": 3.1195, "characteristics.samples_per_second.normalized_per_processor": 374.34, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.25-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 500, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "81447fb30e9ac6ca", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9988839, "90.00 percentile latency (ns)": 12320530, "90th percentile latency (ns)": 12320530, "95.00 percentile latency (ns)": 15244269, "97.00 percentile latency (ns)": 16586074, "99.00 percentile latency (ns)": 24837705, "99.90 percentile latency (ns)": 70659830, "Max latency (ns)": 96220956, "Mean latency (ns)": 10877151, "Min duration satisfied": "Yes", "Min latency (ns)": 9148035, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 91.85, "QPS w/o loadgen overhead": 91.94, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.126, "characteristics.good": 34063, "characteristics.power": 20.429733333333306, "characteristics.power.normalized_per_core": 0.17024777777777755, "characteristics.power.normalized_per_processor": 20.429733333333306, "characteristics.samples_per_second": 91.94, "characteristics.samples_per_second.normalized_per_core": 0.7661666666666667, "characteristics.samples_per_second.normalized_per_processor": 91.94, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "a1b54a984472ba60", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9259048, "90.00 percentile latency (ns)": 11655958, "90th percentile latency (ns)": 11655958, "95.00 percentile latency (ns)": 14440246, "97.00 percentile latency (ns)": 16079392, "99.00 percentile latency (ns)": 24294201, "99.90 percentile latency (ns)": 84365622, "Max latency (ns)": 123385970, "Mean latency (ns)": 10177933, "Min duration satisfied": "Yes", "Min latency (ns)": 8653804, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 98.09, "QPS w/o loadgen overhead": 98.25, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.76, "characteristics.good": 37880, "characteristics.power": 25.135766666666637, "characteristics.power.normalized_per_core": 0.20946472222222198, "characteristics.power.normalized_per_processor": 25.135766666666637, "characteristics.samples_per_second": 98.25, "characteristics.samples_per_second.normalized_per_core": 0.81875, "characteristics.samples_per_second.normalized_per_processor": 98.25, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-large_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6df9e283d544ea7a", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5683855, "90.00 percentile latency (ns)": 7933582, "90th percentile latency (ns)": 7933582, "95.00 percentile latency (ns)": 9762811, "97.00 percentile latency (ns)": 11147510, "99.00 percentile latency (ns)": 13908586, "99.90 percentile latency (ns)": 73532353, "Max latency (ns)": 148892361, "Mean latency (ns)": 6439037, "Min duration satisfied": "Yes", "Min latency (ns)": 4987603, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 154.84, "QPS w/o loadgen overhead": 155.3, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 65.75, "characteristics.good": 32875, "characteristics.power": 24.128316666666695, "characteristics.power.normalized_per_core": 0.2010693055555558, "characteristics.power.normalized_per_processor": 24.128316666666695, "characteristics.samples_per_second": 155.3, "characteristics.samples_per_second.normalized_per_core": 1.2941666666666667, "characteristics.samples_per_second.normalized_per_processor": 155.3, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "1bf94ec444bb6a0c", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6701845, "90.00 percentile latency (ns)": 8383554, "90th percentile latency (ns)": 8383554, "95.00 percentile latency (ns)": 9979500, "97.00 percentile latency (ns)": 11779352, "99.00 percentile latency (ns)": 16705667, "99.90 percentile latency (ns)": 64145641, "Max latency (ns)": 81521707, "Mean latency (ns)": 7312294, "Min duration satisfied": "Yes", "Min latency (ns)": 6026582, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 136.57, "QPS w/o loadgen overhead": 136.76, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.768, "characteristics.good": 32384, "characteristics.power": 20.16076666666667, "characteristics.power.normalized_per_core": 0.1680063888888889, "characteristics.power.normalized_per_processor": 20.16076666666667, "characteristics.samples_per_second": 136.76, "characteristics.samples_per_second.normalized_per_core": 1.1396666666666666, "characteristics.samples_per_second.normalized_per_processor": 136.76, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-1.0-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "37ea452b9727773a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8247874, "90.00 percentile latency (ns)": 13254307, "90th percentile latency (ns)": 13254307, "95.00 percentile latency (ns)": 14718119, "97.00 percentile latency (ns)": 15486069, "99.00 percentile latency (ns)": 23594050, "99.90 percentile latency (ns)": 82583648, "Max latency (ns)": 131854820, "Mean latency (ns)": 9658512, "Min duration satisfied": "Yes", "Min latency (ns)": 7160239, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 103.33, "QPS w/o loadgen overhead": 103.54, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.122, "characteristics.good": 35061, "characteristics.power": 24.728983333333343, "characteristics.power.normalized_per_core": 0.2060748611111112, "characteristics.power.normalized_per_processor": 24.728983333333343, "characteristics.samples_per_second": 103.54, "characteristics.samples_per_second.normalized_per_core": 0.8628333333333333, "characteristics.samples_per_second.normalized_per_processor": 103.54, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6dd46e81599a43ad", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6202502, "90.00 percentile latency (ns)": 7734703, "90th percentile latency (ns)": 7734703, "95.00 percentile latency (ns)": 9641347, "97.00 percentile latency (ns)": 11474500, "99.00 percentile latency (ns)": 14579694, "99.90 percentile latency (ns)": 60247568, "Max latency (ns)": 79071759, "Mean latency (ns)": 6806530, "Min duration satisfied": "Yes", "Min latency (ns)": 5503369, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 146.71, "QPS w/o loadgen overhead": 146.92, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.08, "characteristics.good": 32040, "characteristics.power": 20.27921666666669, "characteristics.power.normalized_per_core": 0.16899347222222244, "characteristics.power.normalized_per_processor": 20.27921666666669, "characteristics.samples_per_second": 146.92, "characteristics.samples_per_second.normalized_per_core": 1.2243333333333333, "characteristics.samples_per_second.normalized_per_processor": 146.92, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "29945fa05943fd93", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4272020, "90.00 percentile latency (ns)": 5016693, "90th percentile latency (ns)": 5016693, "95.00 percentile latency (ns)": 5979871, "97.00 percentile latency (ns)": 7222096, "99.00 percentile latency (ns)": 11026353, "99.90 percentile latency (ns)": 45941030, "Max latency (ns)": 129411756, "Mean latency (ns)": 4662841, "Min duration satisfied": "Yes", "Min latency (ns)": 3737115, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 213.77, "QPS w/o loadgen overhead": 214.46, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.282, "characteristics.good": 32141, "characteristics.power": 23.670200000000012, "characteristics.power.normalized_per_core": 0.19725166666666677, "characteristics.power.normalized_per_processor": 23.670200000000012, "characteristics.samples_per_second": 214.46, "characteristics.samples_per_second.normalized_per_core": 1.7871666666666668, "characteristics.samples_per_second.normalized_per_processor": 214.46, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "627a134f4003b704", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7446929, "90.00 percentile latency (ns)": 9102716, "90th percentile latency (ns)": 9102716, "95.00 percentile latency (ns)": 11797846, "97.00 percentile latency (ns)": 13773039, "99.00 percentile latency (ns)": 18643532, "99.90 percentile latency (ns)": 78123440, "Max latency (ns)": 124748973, "Mean latency (ns)": 8213634, "Min duration satisfied": "Yes", "Min latency (ns)": 6851446, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 121.43, "QPS w/o loadgen overhead": 121.75, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.678, "characteristics.good": 36339, "characteristics.power": 25.238083333333336, "characteristics.power.normalized_per_core": 0.21031736111111113, "characteristics.power.normalized_per_processor": 25.238083333333336, "characteristics.samples_per_second": 121.75, "characteristics.samples_per_second.normalized_per_core": 1.0145833333333334, "characteristics.samples_per_second.normalized_per_processor": 121.75, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "024b109c3f4cb505", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6349189, "90.00 percentile latency (ns)": 9507336, "90th percentile latency (ns)": 9507336, "95.00 percentile latency (ns)": 12269780, "97.00 percentile latency (ns)": 13384653, "99.00 percentile latency (ns)": 16090393, "99.90 percentile latency (ns)": 74508995, "Max latency (ns)": 134802278, "Mean latency (ns)": 7270800, "Min duration satisfied": "Yes", "Min latency (ns)": 5648289, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 137.37, "QPS w/o loadgen overhead": 137.54, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.972, "characteristics.good": 32486, "characteristics.power": 19.982116666666656, "characteristics.power.normalized_per_core": 0.1665176388888888, "characteristics.power.normalized_per_processor": 19.982116666666656, "characteristics.samples_per_second": 137.54, "characteristics.samples_per_second.normalized_per_core": 1.1461666666666666, "characteristics.samples_per_second.normalized_per_processor": 137.54, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c1c7f41775714681", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9166415, "90.00 percentile latency (ns)": 13995445, "90th percentile latency (ns)": 13995445, "95.00 percentile latency (ns)": 15629134, "97.00 percentile latency (ns)": 16509242, "99.00 percentile latency (ns)": 25462351, "99.90 percentile latency (ns)": 86192017, "Max latency (ns)": 127894700, "Mean latency (ns)": 10510643, "Min duration satisfied": "Yes", "Min latency (ns)": 8293926, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 94.97, "QPS w/o loadgen overhead": 95.14, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.77, "characteristics.good": 35885, "characteristics.power": 25.707633333333327, "characteristics.power.normalized_per_core": 0.21423027777777773, "characteristics.power.normalized_per_processor": 25.707633333333327, "characteristics.samples_per_second": 95.14, "characteristics.samples_per_second.normalized_per_core": 0.7928333333333334, "characteristics.samples_per_second.normalized_per_processor": 95.14, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-1.0-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6ac5d8b3422ca3fd", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4933922, "90.00 percentile latency (ns)": 7797315, "90th percentile latency (ns)": 7797315, "95.00 percentile latency (ns)": 9506099, "97.00 percentile latency (ns)": 10729883, "99.00 percentile latency (ns)": 12885174, "99.90 percentile latency (ns)": 68088157, "Max latency (ns)": 129310893, "Mean latency (ns)": 5789391, "Min duration satisfied": "Yes", "Min latency (ns)": 4143552, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 172.14, "QPS w/o loadgen overhead": 172.73, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.75, "characteristics.good": 33375, "characteristics.power": 24.277266666666684, "characteristics.power.normalized_per_core": 0.2023105555555557, "characteristics.power.normalized_per_processor": 24.277266666666684, "characteristics.samples_per_second": 172.73, "characteristics.samples_per_second.normalized_per_core": 1.4394166666666666, "characteristics.samples_per_second.normalized_per_processor": 172.73, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.75-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "8fa5d9679860b9c4", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 1756273, "90.00 percentile latency (ns)": 2016892, "90th percentile latency (ns)": 2016892, "95.00 percentile latency (ns)": 2393389, "97.00 percentile latency (ns)": 2989833, "99.00 percentile latency (ns)": 4708728, "99.90 percentile latency (ns)": 23756256, "Max latency (ns)": 104490008, "Mean latency (ns)": 1919009, "Min duration satisfied": "Yes", "Min latency (ns)": 1310140, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 519.27, "QPS w/o loadgen overhead": 521.1, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 47.14, "characteristics.good": 23570, "characteristics.power": 21.52591666666667, "characteristics.power.normalized_per_core": 0.17938263888888892, "characteristics.power.normalized_per_processor": 21.52591666666667, "characteristics.samples_per_second": 521.1, "characteristics.samples_per_second.normalized_per_core": 4.3425, "characteristics.samples_per_second.normalized_per_processor": 521.1, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "498bb2938bc24bfd", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2918155, "90.00 percentile latency (ns)": 3288427, "90th percentile latency (ns)": 3288427, "95.00 percentile latency (ns)": 3887513, "97.00 percentile latency (ns)": 4458859, "99.00 percentile latency (ns)": 6813340, "99.90 percentile latency (ns)": 33989565, "Max latency (ns)": 127548628, "Mean latency (ns)": 3159451, "Min duration satisfied": "Yes", "Min latency (ns)": 2424800, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 315.22, "QPS w/o loadgen overhead": 316.51, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.988, "characteristics.good": 28494, "characteristics.power": 22.769416666666668, "characteristics.power.normalized_per_core": 0.18974513888888891, "characteristics.power.normalized_per_processor": 22.769416666666668, "characteristics.samples_per_second": 316.51, "characteristics.samples_per_second.normalized_per_core": 2.637583333333333, "characteristics.samples_per_second.normalized_per_processor": 316.51, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "b9fdbf73413495b4", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3073165, "90.00 percentile latency (ns)": 3740908, "90th percentile latency (ns)": 3740908, "95.00 percentile latency (ns)": 4767984, "97.00 percentile latency (ns)": 5522192, "99.00 percentile latency (ns)": 8243549, "99.90 percentile latency (ns)": 39900496, "Max latency (ns)": 80866685, "Mean latency (ns)": 3405683, "Min duration satisfied": "Yes", "Min latency (ns)": 2764482, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 292.48, "QPS w/o loadgen overhead": 293.63, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.716, "characteristics.good": 31858, "characteristics.power": 25.874616666666654, "characteristics.power.normalized_per_core": 0.21562180555555546, "characteristics.power.normalized_per_processor": 25.874616666666654, "characteristics.samples_per_second": 293.63, "characteristics.samples_per_second.normalized_per_core": 2.4469166666666666, "characteristics.samples_per_second.normalized_per_processor": 293.63, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v1-0.75-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6ba93af8be758d4f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3314159, "90.00 percentile latency (ns)": 3877601, "90th percentile latency (ns)": 3877601, "95.00 percentile latency (ns)": 4813913, "97.00 percentile latency (ns)": 5623902, "99.00 percentile latency (ns)": 9230497, "99.90 percentile latency (ns)": 34872865, "Max latency (ns)": 128444115, "Mean latency (ns)": 3648134, "Min duration satisfied": "Yes", "Min latency (ns)": 2894146, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 273.68, "QPS w/o loadgen overhead": 274.11, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 55.54, "characteristics.good": 27770, "characteristics.power": 19.576200000000014, "characteristics.power.normalized_per_core": 0.1631350000000001, "characteristics.power.normalized_per_processor": 19.576200000000014, "characteristics.samples_per_second": 274.11, "characteristics.samples_per_second.normalized_per_core": 2.28425, "characteristics.samples_per_second.normalized_per_processor": 274.11, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.5-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "83856f495f05b9a6", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3589285, "90.00 percentile latency (ns)": 4380112, "90th percentile latency (ns)": 4380112, "95.00 percentile latency (ns)": 5313444, "97.00 percentile latency (ns)": 6337879, "99.00 percentile latency (ns)": 9495915, "99.90 percentile latency (ns)": 36858415, "Max latency (ns)": 133198777, "Mean latency (ns)": 3955704, "Min duration satisfied": "Yes", "Min latency (ns)": 3099381, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 251.71, "QPS w/o loadgen overhead": 252.8, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.322, "characteristics.good": 29661, "characteristics.power": 23.316849999999995, "characteristics.power.normalized_per_core": 0.1943070833333333, "characteristics.power.normalized_per_processor": 23.316849999999995, "characteristics.samples_per_second": 252.8, "characteristics.samples_per_second.normalized_per_core": 2.106666666666667, "characteristics.samples_per_second.normalized_per_processor": 252.8, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "mobilenet-v2-0.35-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "5545ecbf3dacfd40", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 34532907, "90.00 percentile latency (ns)": 39326003, "90th percentile latency (ns)": 39326003, "95.00 percentile latency (ns)": 45953876, "97.00 percentile latency (ns)": 54877565, "99.00 percentile latency (ns)": 98724154, "99.90 percentile latency (ns)": 120949711, "Max latency (ns)": 136593332, "Mean latency (ns)": 36664927, "Min duration satisfied": "Yes", "Min latency (ns)": 33392916, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 27.26, "QPS w/o loadgen overhead": 27.27, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.242, "characteristics.good": 34121, "characteristics.power": 6.891066666666671, "characteristics.power.normalized_per_core": 0.05742555555555559, "characteristics.power.normalized_per_processor": 6.891066666666671, "characteristics.samples_per_second": 27.27, "characteristics.samples_per_second.normalized_per_core": 0.22725, "characteristics.samples_per_second.normalized_per_processor": 27.27, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 27.027, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "b3bec343e6a5f909", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 43773134, "90.00 percentile latency (ns)": 51124924, "90th percentile latency (ns)": 51124924, "95.00 percentile latency (ns)": 59814121, "97.00 percentile latency (ns)": 66967734, "99.00 percentile latency (ns)": 123343971, "99.90 percentile latency (ns)": 169708038, "Max latency (ns)": 191882174, "Mean latency (ns)": 46643029, "Min duration satisfied": "Yes", "Min latency (ns)": 42852081, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 21.43, "QPS w/o loadgen overhead": 21.44, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.738, "characteristics.good": 35369, "characteristics.power": 6.720399999999961, "characteristics.power.normalized_per_core": 0.056003333333333, "characteristics.power.normalized_per_processor": 6.720399999999961, "characteristics.samples_per_second": 21.44, "characteristics.samples_per_second.normalized_per_core": 0.17866666666666667, "characteristics.samples_per_second.normalized_per_processor": 21.44, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 22.2222, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "9ed2bf4b52547b6a", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 162527843, "90.00 percentile latency (ns)": 182276944, "90th percentile latency (ns)": 182276944, "95.00 percentile latency (ns)": 184623681, "97.00 percentile latency (ns)": 186921712, "99.00 percentile latency (ns)": 200297956, "99.90 percentile latency (ns)": 313507522, "Max latency (ns)": 323690308, "Mean latency (ns)": 167799895, "Min duration satisfied": "Yes", "Min latency (ns)": 160589951, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 5.96, "QPS w/o loadgen overhead": 5.96, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 78.514, "characteristics.good": 39257, "characteristics.power": 7.536766666666692, "characteristics.power.normalized_per_core": 0.06280638888888909, "characteristics.power.normalized_per_processor": 7.536766666666692, "characteristics.samples_per_second": 5.96, "characteristics.samples_per_second.normalized_per_core": 0.049666666666666665, "characteristics.samples_per_second.normalized_per_processor": 5.96, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite4-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 5.91716, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "2acfca44f29f65ea", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 12007086, "90.00 percentile latency (ns)": 12854829, "90th percentile latency (ns)": 12854829, "95.00 percentile latency (ns)": 15280328, "97.00 percentile latency (ns)": 18836620, "99.00 percentile latency (ns)": 36646158, "99.90 percentile latency (ns)": 127101501, "Max latency (ns)": 149966587, "Mean latency (ns)": 13009009, "Min duration satisfied": "Yes", "Min latency (ns)": 11189312, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 76.6, "QPS w/o loadgen overhead": 76.87, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.542, "characteristics.good": 31771, "characteristics.power": 7.172316666666643, "characteristics.power.normalized_per_core": 0.05976930555555536, "characteristics.power.normalized_per_processor": 7.172316666666643, "characteristics.samples_per_second": 76.87, "characteristics.samples_per_second.normalized_per_core": 0.6405833333333334, "characteristics.samples_per_second.normalized_per_processor": 76.87, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "0f0f18740ac891ce", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8576081, "90.00 percentile latency (ns)": 9265739, "90th percentile latency (ns)": 9265739, "95.00 percentile latency (ns)": 9942817, "97.00 percentile latency (ns)": 12766893, "99.00 percentile latency (ns)": 27992931, "99.90 percentile latency (ns)": 113449609, "Max latency (ns)": 133533685, "Mean latency (ns)": 9296090, "Min duration satisfied": "Yes", "Min latency (ns)": 7714359, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 106.79, "QPS w/o loadgen overhead": 107.57, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 52.172, "characteristics.good": 26086, "characteristics.power": 6.635416666666659, "characteristics.power.normalized_per_core": 0.055295138888888824, "characteristics.power.normalized_per_processor": 6.635416666666659, "characteristics.samples_per_second": 107.57, "characteristics.samples_per_second.normalized_per_core": 0.8964166666666666, "characteristics.samples_per_second.normalized_per_processor": 107.57, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "af14a501dc1456ab", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10270733, "90.00 percentile latency (ns)": 11133435, "90th percentile latency (ns)": 11133435, "95.00 percentile latency (ns)": 12503942, "97.00 percentile latency (ns)": 15686092, "99.00 percentile latency (ns)": 33600912, "99.90 percentile latency (ns)": 117713711, "Max latency (ns)": 191092379, "Mean latency (ns)": 11184176, "Min duration satisfied": "Yes", "Min latency (ns)": 9476083, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 89.04, "QPS w/o loadgen overhead": 89.41, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.402, "characteristics.good": 29201, "characteristics.power": 6.698866666666658, "characteristics.power.normalized_per_core": 0.05582388888888881, "characteristics.power.normalized_per_processor": 6.698866666666658, "characteristics.samples_per_second": 89.41, "characteristics.samples_per_second.normalized_per_core": 0.7450833333333333, "characteristics.samples_per_second.normalized_per_processor": 89.41, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "2bf5bda347d0b211", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 22509615, "90.00 percentile latency (ns)": 23396379, "90th percentile latency (ns)": 23396379, "95.00 percentile latency (ns)": 27682763, "97.00 percentile latency (ns)": 32576571, "99.00 percentile latency (ns)": 49340640, "99.90 percentile latency (ns)": 138339465, "Max latency (ns)": 154939383, "Mean latency (ns)": 23898280, "Min duration satisfied": "Yes", "Min latency (ns)": 21913670, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 41.78, "QPS w/o loadgen overhead": 41.84, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 60.976, "characteristics.good": 30488, "characteristics.power": 6.348949999999995, "characteristics.power.normalized_per_core": 0.052907916666666624, "characteristics.power.normalized_per_processor": 6.348949999999995, "characteristics.samples_per_second": 41.84, "characteristics.samples_per_second.normalized_per_core": 0.3486666666666667, "characteristics.samples_per_second.normalized_per_processor": 41.84, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 43.4783, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "b10bcc74dc6937ff", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3881768, "90.00 percentile latency (ns)": 4595420, "90th percentile latency (ns)": 4595420, "95.00 percentile latency (ns)": 4803786, "97.00 percentile latency (ns)": 4998022, "99.00 percentile latency (ns)": 12090695, "99.90 percentile latency (ns)": 62218277, "Max latency (ns)": 97802116, "Mean latency (ns)": 4300641, "Min duration satisfied": "Yes", "Min latency (ns)": 3348335, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 230.48, "QPS w/o loadgen overhead": 232.52, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 48.162, "characteristics.good": 24081, "characteristics.power": 7.145600000000001, "characteristics.power.normalized_per_core": 0.05954666666666667, "characteristics.power.normalized_per_processor": 7.145600000000001, "characteristics.samples_per_second": 232.52, "characteristics.samples_per_second.normalized_per_core": 1.9376666666666666, "characteristics.samples_per_second.normalized_per_processor": 232.52, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ba163afa84697a5c", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 68114293, "90.00 percentile latency (ns)": 78796789, "90th percentile latency (ns)": 78796789, "95.00 percentile latency (ns)": 87000206, "97.00 percentile latency (ns)": 88519149, "99.00 percentile latency (ns)": 93004031, "99.90 percentile latency (ns)": 167370756, "Max latency (ns)": 205512779, "Mean latency (ns)": 70434336, "Min duration satisfied": "Yes", "Min latency (ns)": 66763035, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 14.19, "QPS w/o loadgen overhead": 14.2, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.068, "characteristics.good": 37534, "characteristics.power": 6.890683333333341, "characteristics.power.normalized_per_core": 0.057422361111111174, "characteristics.power.normalized_per_processor": 6.890683333333341, "characteristics.samples_per_second": 14.2, "characteristics.samples_per_second.normalized_per_core": 0.11833333333333333, "characteristics.samples_per_second.normalized_per_processor": 14.2, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite0-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.2857, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "0c04ec697daa8f8c", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5010197, "90.00 percentile latency (ns)": 5623287, "90th percentile latency (ns)": 5623287, "95.00 percentile latency (ns)": 5872621, "97.00 percentile latency (ns)": 6443217, "99.00 percentile latency (ns)": 16735749, "99.90 percentile latency (ns)": 64558991, "Max latency (ns)": 108994418, "Mean latency (ns)": 5477952, "Min duration satisfied": "Yes", "Min latency (ns)": 4422893, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 181.1, "QPS w/o loadgen overhead": 182.55, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 50.242, "characteristics.good": 25121, "characteristics.power": 7.135500000000002, "characteristics.power.normalized_per_core": 0.059462500000000015, "characteristics.power.normalized_per_processor": 7.135500000000002, "characteristics.samples_per_second": 182.55, "characteristics.samples_per_second.normalized_per_core": 1.52125, "characteristics.samples_per_second.normalized_per_processor": 182.55, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "bc191cead6155386", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 14008338, "90.00 percentile latency (ns)": 14763228, "90th percentile latency (ns)": 14763228, "95.00 percentile latency (ns)": 17497790, "97.00 percentile latency (ns)": 20131609, "99.00 percentile latency (ns)": 38395913, "99.90 percentile latency (ns)": 85823477, "Max latency (ns)": 113941198, "Mean latency (ns)": 14960283, "Min duration satisfied": "Yes", "Min latency (ns)": 13211273, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 66.73, "QPS w/o loadgen overhead": 66.84, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 60.842, "characteristics.good": 30421, "characteristics.power": 6.854599999999997, "characteristics.power.normalized_per_core": 0.05712166666666664, "characteristics.power.normalized_per_processor": 6.854599999999997, "characteristics.samples_per_second": 66.84, "characteristics.samples_per_second.normalized_per_core": 0.557, "characteristics.samples_per_second.normalized_per_processor": 66.84, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "cbce4e7ea7e286ef", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 32675962, "90.00 percentile latency (ns)": 37019139, "90th percentile latency (ns)": 37019139, "95.00 percentile latency (ns)": 44514868, "97.00 percentile latency (ns)": 50852171, "99.00 percentile latency (ns)": 68343705, "99.90 percentile latency (ns)": 158531009, "Max latency (ns)": 171913564, "Mean latency (ns)": 34921442, "Min duration satisfied": "Yes", "Min latency (ns)": 31802194, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 28.61, "QPS w/o loadgen overhead": 28.64, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.016, "characteristics.good": 34508, "characteristics.power": 6.758133333333356, "characteristics.power.normalized_per_core": 0.05631777777777797, "characteristics.power.normalized_per_processor": 6.758133333333356, "characteristics.samples_per_second": 28.64, "characteristics.samples_per_second.normalized_per_core": 0.23866666666666667, "characteristics.samples_per_second.normalized_per_processor": 28.64, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 28.5714, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "400cb75739d138c0", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 47017344, "90.00 percentile latency (ns)": 52988872, "90th percentile latency (ns)": 52988872, "95.00 percentile latency (ns)": 65916160, "97.00 percentile latency (ns)": 69427968, "99.00 percentile latency (ns)": 73006935, "99.90 percentile latency (ns)": 174037873, "Max latency (ns)": 213172985, "Mean latency (ns)": 48963056, "Min duration satisfied": "Yes", "Min latency (ns)": 45652574, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.41, "QPS w/o loadgen overhead": 20.42, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.098, "characteristics.good": 37049, "characteristics.power": 7.47204999999997, "characteristics.power.normalized_per_core": 0.062267083333333084, "characteristics.power.normalized_per_processor": 7.47204999999997, "characteristics.samples_per_second": 20.42, "characteristics.samples_per_second.normalized_per_core": 0.1701666666666667, "characteristics.samples_per_second.normalized_per_processor": 20.42, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite1-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 20.4082, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "0512d98c99f48d4a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8984906, "90.00 percentile latency (ns)": 9740225, "90th percentile latency (ns)": 9740225, "95.00 percentile latency (ns)": 10883906, "97.00 percentile latency (ns)": 13704323, "99.00 percentile latency (ns)": 30142370, "99.90 percentile latency (ns)": 122017957, "Max latency (ns)": 143097176, "Mean latency (ns)": 9728868, "Min duration satisfied": "Yes", "Min latency (ns)": 8429137, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 102.43, "QPS w/o loadgen overhead": 102.79, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.924, "characteristics.good": 30962, "characteristics.power": 7.130066666666674, "characteristics.power.normalized_per_core": 0.05941722222222228, "characteristics.power.normalized_per_processor": 7.130066666666674, "characteristics.samples_per_second": 102.79, "characteristics.samples_per_second.normalized_per_core": 0.8565833333333334, "characteristics.samples_per_second.normalized_per_processor": 102.79, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "f48cc04d30a50731", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10276326, "90.00 percentile latency (ns)": 11014885, "90th percentile latency (ns)": 11014885, "95.00 percentile latency (ns)": 13102960, "97.00 percentile latency (ns)": 15888231, "99.00 percentile latency (ns)": 34197366, "99.90 percentile latency (ns)": 81389885, "Max latency (ns)": 124497129, "Mean latency (ns)": 11138885, "Min duration satisfied": "Yes", "Min latency (ns)": 9708112, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 89.5, "QPS w/o loadgen overhead": 89.78, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 62.298, "characteristics.good": 31149, "characteristics.power": 7.23731666666669, "characteristics.power.normalized_per_core": 0.06031097222222241, "characteristics.power.normalized_per_processor": 7.23731666666669, "characteristics.samples_per_second": 89.78, "characteristics.samples_per_second.normalized_per_core": 0.7481666666666666, "characteristics.samples_per_second.normalized_per_processor": 89.78, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d344343af360db14", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4138835, "90.00 percentile latency (ns)": 5018704, "90th percentile latency (ns)": 5018704, "95.00 percentile latency (ns)": 5212422, "97.00 percentile latency (ns)": 5422752, "99.00 percentile latency (ns)": 11750595, "99.90 percentile latency (ns)": 93535491, "Max latency (ns)": 132905439, "Mean latency (ns)": 4644423, "Min duration satisfied": "Yes", "Min latency (ns)": 3725641, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 213.31, "QPS w/o loadgen overhead": 215.31, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.12, "characteristics.good": 24560, "characteristics.power": 7.0221166666666885, "characteristics.power.normalized_per_core": 0.05851763888888907, "characteristics.power.normalized_per_processor": 7.0221166666666885, "characteristics.samples_per_second": 215.31, "characteristics.samples_per_second.normalized_per_core": 1.7942500000000001, "characteristics.samples_per_second.normalized_per_processor": 215.31, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "1d381590d294f830", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 39765424, "90.00 percentile latency (ns)": 46728142, "90th percentile latency (ns)": 46728142, "95.00 percentile latency (ns)": 56157089, "97.00 percentile latency (ns)": 62053207, "99.00 percentile latency (ns)": 88323106, "99.90 percentile latency (ns)": 170060321, "Max latency (ns)": 181621239, "Mean latency (ns)": 42294699, "Min duration satisfied": "Yes", "Min latency (ns)": 38417804, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 23.63, "QPS w/o loadgen overhead": 23.64, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.584, "characteristics.good": 36792, "characteristics.power": 7.358733333333324, "characteristics.power.normalized_per_core": 0.0613227777777777, "characteristics.power.normalized_per_processor": 7.358733333333324, "characteristics.samples_per_second": 23.64, "characteristics.samples_per_second.normalized_per_core": 0.197, "characteristics.samples_per_second.normalized_per_processor": 23.64, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.4-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.4-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_140.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 24.3902, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "43cbb72e71d3f2d4", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 28526194, "90.00 percentile latency (ns)": 32052356, "90th percentile latency (ns)": 32052356, "95.00 percentile latency (ns)": 39998975, "97.00 percentile latency (ns)": 47701080, "99.00 percentile latency (ns)": 74037843, "99.90 percentile latency (ns)": 117582515, "Max latency (ns)": 128979463, "Mean latency (ns)": 30419910, "Min duration satisfied": "Yes", "Min latency (ns)": 27629770, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 32.84, "QPS w/o loadgen overhead": 32.87, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.772, "characteristics.good": 35386, "characteristics.power": 7.358716666666662, "characteristics.power.normalized_per_core": 0.06132263888888885, "characteristics.power.normalized_per_processor": 7.358716666666662, "characteristics.samples_per_second": 32.87, "characteristics.samples_per_second.normalized_per_core": 0.27391666666666664, "characteristics.samples_per_second.normalized_per_processor": 32.87, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 32.2581, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "07a82e054568a924", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7099370, "90.00 percentile latency (ns)": 7383863, "90th percentile latency (ns)": 7383863, "95.00 percentile latency (ns)": 7712412, "97.00 percentile latency (ns)": 9051495, "99.00 percentile latency (ns)": 20210444, "99.90 percentile latency (ns)": 65845855, "Max latency (ns)": 87378124, "Mean latency (ns)": 7584050, "Min duration satisfied": "Yes", "Min latency (ns)": 6570139, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 131.55, "QPS w/o loadgen overhead": 131.86, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 47.808, "characteristics.good": 23904, "characteristics.power": 6.611933333333328, "characteristics.power.normalized_per_core": 0.055099444444444395, "characteristics.power.normalized_per_processor": 6.611933333333328, "characteristics.samples_per_second": 131.86, "characteristics.samples_per_second.normalized_per_core": 1.0988333333333336, "characteristics.samples_per_second.normalized_per_processor": 131.86, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "246b63e50dc40f2b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 16839929, "90.00 percentile latency (ns)": 17821295, "90th percentile latency (ns)": 17821295, "95.00 percentile latency (ns)": 21972200, "97.00 percentile latency (ns)": 27797716, "99.00 percentile latency (ns)": 43438713, "99.90 percentile latency (ns)": 134727386, "Max latency (ns)": 169433832, "Mean latency (ns)": 18144616, "Min duration satisfied": "Yes", "Min latency (ns)": 15903748, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 54.99, "QPS w/o loadgen overhead": 55.11, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.038, "characteristics.good": 32019, "characteristics.power": 6.733116666666668, "characteristics.power.normalized_per_core": 0.05610930555555557, "characteristics.power.normalized_per_processor": 6.733116666666668, "characteristics.samples_per_second": 55.11, "characteristics.samples_per_second.normalized_per_core": 0.45925, "characteristics.samples_per_second.normalized_per_processor": 55.11, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 55.5556, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "16a260985811c93d", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 13245614, "90.00 percentile latency (ns)": 13646700, "90th percentile latency (ns)": 13646700, "95.00 percentile latency (ns)": 14244816, "97.00 percentile latency (ns)": 16398472, "99.00 percentile latency (ns)": 38559352, "99.90 percentile latency (ns)": 83205152, "Max latency (ns)": 100191371, "Mean latency (ns)": 14007950, "Min duration satisfied": "Yes", "Min latency (ns)": 12576368, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 71.27, "QPS w/o loadgen overhead": 71.39, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 51.848, "characteristics.good": 25924, "characteristics.power": 6.289333333333347, "characteristics.power.normalized_per_core": 0.05241111111111122, "characteristics.power.normalized_per_processor": 6.289333333333347, "characteristics.samples_per_second": 71.39, "characteristics.samples_per_second.normalized_per_core": 0.5949166666666666, "characteristics.samples_per_second.normalized_per_processor": 71.39, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "f904934ca7bb903a", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6977163, "90.00 percentile latency (ns)": 7320877, "90th percentile latency (ns)": 7320877, "95.00 percentile latency (ns)": 7654554, "97.00 percentile latency (ns)": 10741339, "99.00 percentile latency (ns)": 21305293, "99.90 percentile latency (ns)": 111350419, "Max latency (ns)": 137543602, "Mean latency (ns)": 7512169, "Min duration satisfied": "Yes", "Min latency (ns)": 6462432, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 132.79, "QPS w/o loadgen overhead": 133.12, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.226, "characteristics.good": 29613, "characteristics.power": 7.109150000000001, "characteristics.power.normalized_per_core": 0.05924291666666668, "characteristics.power.normalized_per_processor": 7.109150000000001, "characteristics.samples_per_second": 133.12, "characteristics.samples_per_second.normalized_per_core": 1.1093333333333333, "characteristics.samples_per_second.normalized_per_processor": 133.12, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "08560bf0165c2ccb", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 15791993, "90.00 percentile latency (ns)": 16584791, "90th percentile latency (ns)": 16584791, "95.00 percentile latency (ns)": 20937862, "97.00 percentile latency (ns)": 26775939, "99.00 percentile latency (ns)": 41226422, "99.90 percentile latency (ns)": 88537686, "Max latency (ns)": 128347823, "Mean latency (ns)": 16971607, "Min duration satisfied": "Yes", "Min latency (ns)": 15018510, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 58.83, "QPS w/o loadgen overhead": 58.92, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.248, "characteristics.good": 34124, "characteristics.power": 7.3336833333333225, "characteristics.power.normalized_per_core": 0.061114027777777684, "characteristics.power.normalized_per_processor": 7.3336833333333225, "characteristics.samples_per_second": 58.92, "characteristics.samples_per_second.normalized_per_core": 0.491, "characteristics.samples_per_second.normalized_per_processor": 58.92, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 55.5556, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4b42aa44ff8b136e", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6217528, "90.00 percentile latency (ns)": 6561392, "90th percentile latency (ns)": 6561392, "95.00 percentile latency (ns)": 6796458, "97.00 percentile latency (ns)": 9169612, "99.00 percentile latency (ns)": 17580889, "99.90 percentile latency (ns)": 113262179, "Max latency (ns)": 142245332, "Mean latency (ns)": 6700219, "Min duration satisfied": "Yes", "Min latency (ns)": 5747502, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 148.79, "QPS w/o loadgen overhead": 149.25, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.12, "characteristics.good": 29560, "characteristics.power": 7.09848333333332, "characteristics.power.normalized_per_core": 0.05915402777777767, "characteristics.power.normalized_per_processor": 7.09848333333332, "characteristics.samples_per_second": 149.25, "characteristics.samples_per_second.normalized_per_core": 1.24375, "characteristics.samples_per_second.normalized_per_processor": 149.25, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "186c7a2993141258", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6965805, "90.00 percentile latency (ns)": 7642010, "90th percentile latency (ns)": 7642010, "95.00 percentile latency (ns)": 7991353, "97.00 percentile latency (ns)": 10915038, "99.00 percentile latency (ns)": 24474601, "99.90 percentile latency (ns)": 70594598, "Max latency (ns)": 106032218, "Mean latency (ns)": 7613862, "Min duration satisfied": "Yes", "Min latency (ns)": 6437930, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 130.72, "QPS w/o loadgen overhead": 131.34, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.46, "characteristics.good": 29230, "characteristics.power": 7.212999999999993, "characteristics.power.normalized_per_core": 0.06010833333333328, "characteristics.power.normalized_per_processor": 7.212999999999993, "characteristics.samples_per_second": 131.34, "characteristics.samples_per_second.normalized_per_core": 1.0945, "characteristics.samples_per_second.normalized_per_processor": 131.34, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ded78b8a32443eba", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 26134575, "90.00 percentile latency (ns)": 27921923, "90th percentile latency (ns)": 27921923, "95.00 percentile latency (ns)": 36715841, "97.00 percentile latency (ns)": 44841107, "99.00 percentile latency (ns)": 54223123, "99.90 percentile latency (ns)": 139575433, "Max latency (ns)": 159080482, "Mean latency (ns)": 27993647, "Min duration satisfied": "Yes", "Min latency (ns)": 25340018, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 35.68, "QPS w/o loadgen overhead": 35.72, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.91, "characteristics.good": 36455, "characteristics.power": 7.21884999999999, "characteristics.power.normalized_per_core": 0.06015708333333325, "characteristics.power.normalized_per_processor": 7.21884999999999, "characteristics.samples_per_second": 35.72, "characteristics.samples_per_second.normalized_per_core": 0.29766666666666663, "characteristics.samples_per_second.normalized_per_processor": 35.72, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 35.7143, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ee65ff4e47839975", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 24078488, "90.00 percentile latency (ns)": 25605844, "90th percentile latency (ns)": 25605844, "95.00 percentile latency (ns)": 33434938, "97.00 percentile latency (ns)": 41040640, "99.00 percentile latency (ns)": 51771559, "99.90 percentile latency (ns)": 146298114, "Max latency (ns)": 165208591, "Mean latency (ns)": 25872950, "Min duration satisfied": "Yes", "Min latency (ns)": 23325692, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 38.59, "QPS w/o loadgen overhead": 38.65, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.542, "characteristics.good": 35271, "characteristics.power": 7.312950000000001, "characteristics.power.normalized_per_core": 0.06094125000000001, "characteristics.power.normalized_per_processor": 7.312950000000001, "characteristics.samples_per_second": 38.65, "characteristics.samples_per_second.normalized_per_core": 0.32208333333333333, "characteristics.samples_per_second.normalized_per_processor": 38.65, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 38.4615, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "5635b285b4c71b79", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7258242, "90.00 percentile latency (ns)": 7866320, "90th percentile latency (ns)": 7866320, "95.00 percentile latency (ns)": 8260962, "97.00 percentile latency (ns)": 10602765, "99.00 percentile latency (ns)": 22139822, "99.90 percentile latency (ns)": 117789005, "Max latency (ns)": 136487280, "Mean latency (ns)": 7851212, "Min duration satisfied": "Yes", "Min latency (ns)": 6617448, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 126.37, "QPS w/o loadgen overhead": 127.37, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.594, "characteristics.good": 28297, "characteristics.power": 7.12635000000001, "characteristics.power.normalized_per_core": 0.059386250000000085, "characteristics.power.normalized_per_processor": 7.12635000000001, "characteristics.samples_per_second": 127.37, "characteristics.samples_per_second.normalized_per_core": 1.0614166666666667, "characteristics.samples_per_second.normalized_per_processor": 127.37, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "47b41659a500b6f0", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 13587144, "90.00 percentile latency (ns)": 14392794, "90th percentile latency (ns)": 14392794, "95.00 percentile latency (ns)": 17830737, "97.00 percentile latency (ns)": 22235712, "99.00 percentile latency (ns)": 39318930, "99.90 percentile latency (ns)": 128975627, "Max latency (ns)": 163752265, "Mean latency (ns)": 14670262, "Min duration satisfied": "Yes", "Min latency (ns)": 12726887, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 67.99, "QPS w/o loadgen overhead": 68.17, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.82, "characteristics.good": 33910, "characteristics.power": 7.2364666666666855, "characteristics.power.normalized_per_core": 0.060303888888889046, "characteristics.power.normalized_per_processor": 7.2364666666666855, "characteristics.samples_per_second": 68.17, "characteristics.samples_per_second.normalized_per_core": 0.5680833333333334, "characteristics.samples_per_second.normalized_per_processor": 68.17, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "7e54549bf5775978", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4743252, "90.00 percentile latency (ns)": 5373071, "90th percentile latency (ns)": 5373071, "95.00 percentile latency (ns)": 5660269, "97.00 percentile latency (ns)": 6284720, "99.00 percentile latency (ns)": 13852577, "99.90 percentile latency (ns)": 61487127, "Max latency (ns)": 82249947, "Mean latency (ns)": 5154507, "Min duration satisfied": "Yes", "Min latency (ns)": 4195226, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 192.2, "QPS w/o loadgen overhead": 194.0, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 44.116, "characteristics.good": 22058, "characteristics.power": 6.796050000000004, "characteristics.power.normalized_per_core": 0.05663375000000003, "characteristics.power.normalized_per_processor": 6.796050000000004, "characteristics.samples_per_second": 194.0, "characteristics.samples_per_second.normalized_per_core": 1.6166666666666667, "characteristics.samples_per_second.normalized_per_processor": 194.0, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "743b1a705ca50386", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2929262, "90.00 percentile latency (ns)": 3529894, "90th percentile latency (ns)": 3529894, "95.00 percentile latency (ns)": 3756255, "97.00 percentile latency (ns)": 3926501, "99.00 percentile latency (ns)": 7906950, "99.90 percentile latency (ns)": 60054452, "Max latency (ns)": 113231978, "Mean latency (ns)": 3267606, "Min duration satisfied": "Yes", "Min latency (ns)": 2430200, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 302.15, "QPS w/o loadgen overhead": 306.03, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 45.494, "characteristics.good": 22747, "characteristics.power": 7.124400000000009, "characteristics.power.normalized_per_core": 0.059370000000000075, "characteristics.power.normalized_per_processor": 7.124400000000009, "characteristics.samples_per_second": 306.03, "characteristics.samples_per_second.normalized_per_core": 2.5502499999999997, "characteristics.samples_per_second.normalized_per_processor": 306.03, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "572a51d47f1833ef", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 53369721, "90.00 percentile latency (ns)": 62038220, "90th percentile latency (ns)": 62038220, "95.00 percentile latency (ns)": 69507677, "97.00 percentile latency (ns)": 74047876, "99.00 percentile latency (ns)": 127843216, "99.90 percentile latency (ns)": 146172466, "Max latency (ns)": 188733811, "Mean latency (ns)": 56337011, "Min duration satisfied": "Yes", "Min latency (ns)": 51446806, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.74, "QPS w/o loadgen overhead": 17.75, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.626, "characteristics.good": 35313, "characteristics.power": 6.973916666666653, "characteristics.power.normalized_per_core": 0.05811597222222211, "characteristics.power.normalized_per_processor": 6.973916666666653, "characteristics.samples_per_second": 17.75, "characteristics.samples_per_second.normalized_per_core": 0.14791666666666667, "characteristics.samples_per_second.normalized_per_processor": 17.75, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 17.8571, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6868f28b3aeaf933", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9132997, "90.00 percentile latency (ns)": 9970741, "90th percentile latency (ns)": 9970741, "95.00 percentile latency (ns)": 10807566, "97.00 percentile latency (ns)": 14134484, "99.00 percentile latency (ns)": 29819753, "99.90 percentile latency (ns)": 122209398, "Max latency (ns)": 145804861, "Mean latency (ns)": 9939874, "Min duration satisfied": "Yes", "Min latency (ns)": 8594025, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 100.12, "QPS w/o loadgen overhead": 100.6, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.454, "characteristics.good": 31727, "characteristics.power": 7.201699999999977, "characteristics.power.normalized_per_core": 0.06001416666666647, "characteristics.power.normalized_per_processor": 7.201699999999977, "characteristics.samples_per_second": 100.6, "characteristics.samples_per_second.normalized_per_core": 0.8383333333333333, "characteristics.samples_per_second.normalized_per_processor": 100.6, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "2564b0ab0cdb40c5", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 21354059, "90.00 percentile latency (ns)": 22828359, "90th percentile latency (ns)": 22828359, "95.00 percentile latency (ns)": 29568653, "97.00 percentile latency (ns)": 35588204, "99.00 percentile latency (ns)": 48515621, "99.90 percentile latency (ns)": 104193507, "Max latency (ns)": 138614182, "Mean latency (ns)": 22802326, "Min duration satisfied": "Yes", "Min latency (ns)": 20445284, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 43.77, "QPS w/o loadgen overhead": 43.86, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.04, "characteristics.good": 35020, "characteristics.power": 7.321566666666669, "characteristics.power.normalized_per_core": 0.06101305555555558, "characteristics.power.normalized_per_processor": 7.321566666666669, "characteristics.samples_per_second": 43.86, "characteristics.samples_per_second.normalized_per_core": 0.3655, "characteristics.samples_per_second.normalized_per_processor": 43.86, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 43.4783, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "e24125a8780caf23", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 64086092, "90.00 percentile latency (ns)": 76689744, "90th percentile latency (ns)": 76689744, "95.00 percentile latency (ns)": 85120458, "97.00 percentile latency (ns)": 86653321, "99.00 percentile latency (ns)": 90332165, "99.90 percentile latency (ns)": 185866436, "Max latency (ns)": 223747234, "Mean latency (ns)": 66580278, "Min duration satisfied": "Yes", "Min latency (ns)": 62847825, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 15.01, "QPS w/o loadgen overhead": 15.02, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.124, "characteristics.good": 37062, "characteristics.power": 7.494133333333346, "characteristics.power.normalized_per_core": 0.06245111111111122, "characteristics.power.normalized_per_processor": 7.494133333333346, "characteristics.samples_per_second": 15.02, "characteristics.samples_per_second.normalized_per_core": 0.12516666666666668, "characteristics.samples_per_second.normalized_per_processor": 15.02, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite2-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.9254, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4f13487f80de0e13", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5950737, "90.00 percentile latency (ns)": 6592612, "90th percentile latency (ns)": 6592612, "95.00 percentile latency (ns)": 6821533, "97.00 percentile latency (ns)": 8796007, "99.00 percentile latency (ns)": 20514546, "99.90 percentile latency (ns)": 66687941, "Max latency (ns)": 114782860, "Mean latency (ns)": 6500467, "Min duration satisfied": "Yes", "Min latency (ns)": 5345067, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 152.81, "QPS w/o loadgen overhead": 153.84, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.444, "characteristics.good": 29722, "characteristics.power": 7.202883333333311, "characteristics.power.normalized_per_core": 0.06002402777777759, "characteristics.power.normalized_per_processor": 7.202883333333311, "characteristics.samples_per_second": 153.84, "characteristics.samples_per_second.normalized_per_core": 1.282, "characteristics.samples_per_second.normalized_per_processor": 153.84, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "278312a449a996fc", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7948122, "90.00 percentile latency (ns)": 8540090, "90th percentile latency (ns)": 8540090, "95.00 percentile latency (ns)": 9007118, "97.00 percentile latency (ns)": 12204989, "99.00 percentile latency (ns)": 28324605, "99.90 percentile latency (ns)": 71846772, "Max latency (ns)": 110721548, "Mean latency (ns)": 8620162, "Min duration satisfied": "Yes", "Min latency (ns)": 7315768, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 115.52, "QPS w/o loadgen overhead": 116.01, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.456, "characteristics.good": 30728, "characteristics.power": 7.200699999999985, "characteristics.power.normalized_per_core": 0.06000583333333321, "characteristics.power.normalized_per_processor": 7.200699999999985, "characteristics.samples_per_second": 116.01, "characteristics.samples_per_second.normalized_per_core": 0.96675, "characteristics.samples_per_second.normalized_per_processor": 116.01, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "eda2b475c86695db", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9326776, "90.00 percentile latency (ns)": 10030409, "90th percentile latency (ns)": 10030409, "95.00 percentile latency (ns)": 10898026, "97.00 percentile latency (ns)": 13847856, "99.00 percentile latency (ns)": 30398256, "99.90 percentile latency (ns)": 121367529, "Max latency (ns)": 207652511, "Mean latency (ns)": 10075047, "Min duration satisfied": "Yes", "Min latency (ns)": 8695978, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 98.7, "QPS w/o loadgen overhead": 99.26, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.528, "characteristics.good": 29264, "characteristics.power": 7.130066666666665, "characteristics.power.normalized_per_core": 0.05941722222222221, "characteristics.power.normalized_per_processor": 7.130066666666665, "characteristics.samples_per_second": 99.26, "characteristics.samples_per_second.normalized_per_core": 0.8271666666666667, "characteristics.samples_per_second.normalized_per_processor": 99.26, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "bd12334827a14390", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 22602994, "90.00 percentile latency (ns)": 24193087, "90th percentile latency (ns)": 24193087, "95.00 percentile latency (ns)": 31625698, "97.00 percentile latency (ns)": 37766759, "99.00 percentile latency (ns)": 50554493, "99.90 percentile latency (ns)": 140755725, "Max latency (ns)": 160104617, "Mean latency (ns)": 24281553, "Min duration satisfied": "Yes", "Min latency (ns)": 21709733, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 41.08, "QPS w/o loadgen overhead": 41.18, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.036, "characteristics.good": 33018, "characteristics.power": 6.709816666666643, "characteristics.power.normalized_per_core": 0.05591513888888869, "characteristics.power.normalized_per_processor": 6.709816666666643, "characteristics.samples_per_second": 41.18, "characteristics.samples_per_second.normalized_per_core": 0.3431666666666667, "characteristics.samples_per_second.normalized_per_processor": 41.18, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 43.4783, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "cc83906497580435", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 97960768, "90.00 percentile latency (ns)": 115439688, "90th percentile latency (ns)": 115439688, "95.00 percentile latency (ns)": 118965386, "97.00 percentile latency (ns)": 121057450, "99.00 percentile latency (ns)": 128030163, "99.90 percentile latency (ns)": 242089420, "Max latency (ns)": 250956886, "Mean latency (ns)": 101354147, "Min duration satisfied": "Yes", "Min latency (ns)": 96379397, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 9.86, "QPS w/o loadgen overhead": 9.87, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.642, "characteristics.good": 38321, "characteristics.power": 6.947866666666641, "characteristics.power.normalized_per_core": 0.05789888888888868, "characteristics.power.normalized_per_processor": 6.947866666666641, "characteristics.samples_per_second": 9.87, "characteristics.samples_per_second.normalized_per_core": 0.08224999999999999, "characteristics.samples_per_second.normalized_per_processor": 9.87, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite1-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 10.101, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "e877644fc9b7c637", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 18259725, "90.00 percentile latency (ns)": 19213259, "90th percentile latency (ns)": 19213259, "95.00 percentile latency (ns)": 24189772, "97.00 percentile latency (ns)": 30155406, "99.00 percentile latency (ns)": 44343796, "99.90 percentile latency (ns)": 139545153, "Max latency (ns)": 212642500, "Mean latency (ns)": 19651633, "Min duration satisfied": "Yes", "Min latency (ns)": 17592444, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 50.72, "QPS w/o loadgen overhead": 50.89, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.604, "characteristics.good": 34802, "characteristics.power": 7.278883333333353, "characteristics.power.normalized_per_core": 0.06065736111111128, "characteristics.power.normalized_per_processor": 7.278883333333353, "characteristics.samples_per_second": 50.89, "characteristics.samples_per_second.normalized_per_core": 0.4240833333333333, "characteristics.samples_per_second.normalized_per_processor": 50.89, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 52.6316, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "72a7041b4d4da48d", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 45980817, "90.00 percentile latency (ns)": 52956286, "90th percentile latency (ns)": 52956286, "95.00 percentile latency (ns)": 61841836, "97.00 percentile latency (ns)": 67040577, "99.00 percentile latency (ns)": 118493914, "99.90 percentile latency (ns)": 135724867, "Max latency (ns)": 169987058, "Mean latency (ns)": 48658241, "Min duration satisfied": "Yes", "Min latency (ns)": 44826728, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.55, "QPS w/o loadgen overhead": 20.55, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.262, "characteristics.good": 34631, "characteristics.power": 6.879333333333345, "characteristics.power.normalized_per_core": 0.05732777777777787, "characteristics.power.normalized_per_processor": 6.879333333333345, "characteristics.samples_per_second": 20.55, "characteristics.samples_per_second.normalized_per_core": 0.17125, "characteristics.samples_per_second.normalized_per_processor": 20.55, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 20, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "5586b281a26bd812", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 19821783, "90.00 percentile latency (ns)": 20388364, "90th percentile latency (ns)": 20388364, "95.00 percentile latency (ns)": 26178361, "97.00 percentile latency (ns)": 32519657, "99.00 percentile latency (ns)": 45494375, "99.90 percentile latency (ns)": 138629103, "Max latency (ns)": 173660990, "Mean latency (ns)": 21226835, "Min duration satisfied": "Yes", "Min latency (ns)": 19057932, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 47.07, "QPS w/o loadgen overhead": 47.11, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.734, "characteristics.good": 34367, "characteristics.power": 7.280283333333352, "characteristics.power.normalized_per_core": 0.06066902777777793, "characteristics.power.normalized_per_processor": 7.280283333333352, "characteristics.samples_per_second": 47.11, "characteristics.samples_per_second.normalized_per_core": 0.39258333333333334, "characteristics.samples_per_second.normalized_per_processor": 47.11, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 47.619, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "0fdfb43f99331b45", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 16347276, "90.00 percentile latency (ns)": 16836781, "90th percentile latency (ns)": 16836781, "95.00 percentile latency (ns)": 18620591, "97.00 percentile latency (ns)": 21462178, "99.00 percentile latency (ns)": 41224355, "99.90 percentile latency (ns)": 95788116, "Max latency (ns)": 111889290, "Mean latency (ns)": 17281828, "Min duration satisfied": "Yes", "Min latency (ns)": 15775379, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 57.79, "QPS w/o loadgen overhead": 57.86, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.33, "characteristics.good": 31665, "characteristics.power": 6.362966666666661, "characteristics.power.normalized_per_core": 0.053024722222222175, "characteristics.power.normalized_per_processor": 6.362966666666661, "characteristics.samples_per_second": 57.86, "characteristics.samples_per_second.normalized_per_core": 0.4821666666666667, "characteristics.samples_per_second.normalized_per_processor": 57.86, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 58.8235, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "46dc34af7b8a80e4", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9394916, "90.00 percentile latency (ns)": 10055227, "90th percentile latency (ns)": 10055227, "95.00 percentile latency (ns)": 11281088, "97.00 percentile latency (ns)": 14857920, "99.00 percentile latency (ns)": 33912504, "99.90 percentile latency (ns)": 72943922, "Max latency (ns)": 98971866, "Mean latency (ns)": 10189164, "Min duration satisfied": "Yes", "Min latency (ns)": 8580731, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 97.81, "QPS w/o loadgen overhead": 98.14, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 57.918, "characteristics.good": 28959, "characteristics.power": 6.911183333333323, "characteristics.power.normalized_per_core": 0.05759319444444436, "characteristics.power.normalized_per_processor": 6.911183333333323, "characteristics.samples_per_second": 98.14, "characteristics.samples_per_second.normalized_per_core": 0.8178333333333333, "characteristics.samples_per_second.normalized_per_processor": 98.14, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "da8d28ded5cdb9ac", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 67937619, "90.00 percentile latency (ns)": 78942707, "90th percentile latency (ns)": 78942707, "95.00 percentile latency (ns)": 86021029, "97.00 percentile latency (ns)": 93294868, "99.00 percentile latency (ns)": 147890276, "99.90 percentile latency (ns)": 167699131, "Max latency (ns)": 173680561, "Mean latency (ns)": 71510085, "Min duration satisfied": "Yes", "Min latency (ns)": 66088362, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 13.98, "QPS w/o loadgen overhead": 13.98, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.676, "characteristics.good": 35838, "characteristics.power": 7.04420000000001, "characteristics.power.normalized_per_core": 0.05870166666666675, "characteristics.power.normalized_per_processor": 7.04420000000001, "characteristics.samples_per_second": 13.98, "characteristics.samples_per_second.normalized_per_core": 0.1165, "characteristics.samples_per_second.normalized_per_processor": 13.98, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.0845, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "9cd616374fc79c43", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3010062, "90.00 percentile latency (ns)": 3596198, "90th percentile latency (ns)": 3596198, "95.00 percentile latency (ns)": 3761915, "97.00 percentile latency (ns)": 3908357, "99.00 percentile latency (ns)": 7884633, "99.90 percentile latency (ns)": 54375709, "Max latency (ns)": 126305450, "Mean latency (ns)": 3331624, "Min duration satisfied": "Yes", "Min latency (ns)": 2460260, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 295.85, "QPS w/o loadgen overhead": 300.15, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 43.468, "characteristics.good": 21734, "characteristics.power": 6.953128119800307, "characteristics.power.normalized_per_core": 0.05794273433166922, "characteristics.power.normalized_per_processor": 6.953128119800307, "characteristics.samples_per_second": 300.15, "characteristics.samples_per_second.normalized_per_core": 2.5012499999999998, "characteristics.samples_per_second.normalized_per_processor": 300.15, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "265899280b05e246", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 333924933, "90.00 percentile latency (ns)": 352801253, "90th percentile latency (ns)": 352801253, "95.00 percentile latency (ns)": 362539452, "97.00 percentile latency (ns)": 365196629, "99.00 percentile latency (ns)": 374068175, "99.90 percentile latency (ns)": 496080162, "Max latency (ns)": 505116722, "Mean latency (ns)": 341057531, "Min duration satisfied": "Yes", "Min latency (ns)": 328901548, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.93, "QPS w/o loadgen overhead": 2.93, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 80.676, "characteristics.good": 40338, "characteristics.power": 7.15527454242928, "characteristics.power.normalized_per_core": 0.05962728785357733, "characteristics.power.normalized_per_processor": 7.15527454242928, "characteristics.samples_per_second": 2.93, "characteristics.samples_per_second.normalized_per_core": 0.024416666666666666, "characteristics.samples_per_second.normalized_per_processor": 2.93, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite4-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.92398, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "9424af1874733df0", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 18691472, "90.00 percentile latency (ns)": 19529162, "90th percentile latency (ns)": 19529162, "95.00 percentile latency (ns)": 24637301, "97.00 percentile latency (ns)": 30302573, "99.00 percentile latency (ns)": 43941439, "99.90 percentile latency (ns)": 126977153, "Max latency (ns)": 144543687, "Mean latency (ns)": 20039140, "Min duration satisfied": "Yes", "Min latency (ns)": 17962696, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 49.79, "QPS w/o loadgen overhead": 49.9, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.43, "characteristics.good": 35215, "characteristics.power": 7.260116666666692, "characteristics.power.normalized_per_core": 0.06050097222222243, "characteristics.power.normalized_per_processor": 7.260116666666692, "characteristics.samples_per_second": 49.9, "characteristics.samples_per_second.normalized_per_core": 0.41583333333333333, "characteristics.samples_per_second.normalized_per_processor": 49.9, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "1d2a89a80e7b023e", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 15124990, "90.00 percentile latency (ns)": 16032379, "90th percentile latency (ns)": 16032379, "95.00 percentile latency (ns)": 19827590, "97.00 percentile latency (ns)": 24885007, "99.00 percentile latency (ns)": 40224033, "99.90 percentile latency (ns)": 132115389, "Max latency (ns)": 149317204, "Mean latency (ns)": 16329790, "Min duration satisfied": "Yes", "Min latency (ns)": 14234105, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 61.09, "QPS w/o loadgen overhead": 61.24, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.474, "characteristics.good": 33737, "characteristics.power": 7.235233333333346, "characteristics.power.normalized_per_core": 0.060293611111111214, "characteristics.power.normalized_per_processor": 7.235233333333346, "characteristics.samples_per_second": 61.24, "characteristics.samples_per_second.normalized_per_core": 0.5103333333333333, "characteristics.samples_per_second.normalized_per_processor": 61.24, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 62.5, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "5f8d9e1f5a5eea98", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 13639736, "90.00 percentile latency (ns)": 14444997, "90th percentile latency (ns)": 14444997, "95.00 percentile latency (ns)": 18159507, "97.00 percentile latency (ns)": 22849868, "99.00 percentile latency (ns)": 37835130, "99.90 percentile latency (ns)": 88483865, "Max latency (ns)": 133366342, "Mean latency (ns)": 14756086, "Min duration satisfied": "Yes", "Min latency (ns)": 12963288, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 67.61, "QPS w/o loadgen overhead": 67.77, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.24, "characteristics.good": 33620, "characteristics.power": 7.328749999999996, "characteristics.power.normalized_per_core": 0.06107291666666663, "characteristics.power.normalized_per_processor": 7.328749999999996, "characteristics.samples_per_second": 67.77, "characteristics.samples_per_second.normalized_per_core": 0.56475, "characteristics.samples_per_second.normalized_per_processor": 67.77, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "92cba2bdbd4d2f92", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 25806272, "90.00 percentile latency (ns)": 27311020, "90th percentile latency (ns)": 27311020, "95.00 percentile latency (ns)": 33868249, "97.00 percentile latency (ns)": 41827418, "99.00 percentile latency (ns)": 77262259, "99.90 percentile latency (ns)": 101795999, "Max latency (ns)": 118290725, "Mean latency (ns)": 27398853, "Min duration satisfied": "Yes", "Min latency (ns)": 25105767, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 36.47, "QPS w/o loadgen overhead": 36.5, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.348, "characteristics.good": 32174, "characteristics.power": 6.721066666666643, "characteristics.power.normalized_per_core": 0.05600888888888869, "characteristics.power.normalized_per_processor": 6.721066666666643, "characteristics.samples_per_second": 36.5, "characteristics.samples_per_second.normalized_per_core": 0.30416666666666664, "characteristics.samples_per_second.normalized_per_processor": 36.5, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 35.7143, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "404090423a26b330", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 19053132, "90.00 percentile latency (ns)": 20049574, "90th percentile latency (ns)": 20049574, "95.00 percentile latency (ns)": 23356756, "97.00 percentile latency (ns)": 28000985, "99.00 percentile latency (ns)": 46920188, "99.90 percentile latency (ns)": 93180731, "Max latency (ns)": 112891740, "Mean latency (ns)": 20266661, "Min duration satisfied": "Yes", "Min latency (ns)": 18494481, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 49.28, "QPS w/o loadgen overhead": 49.34, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.246, "characteristics.good": 31623, "characteristics.power": 6.755750000000023, "characteristics.power.normalized_per_core": 0.05629791666666686, "characteristics.power.normalized_per_processor": 6.755750000000023, "characteristics.samples_per_second": 49.34, "characteristics.samples_per_second.normalized_per_core": 0.4111666666666667, "characteristics.samples_per_second.normalized_per_processor": 49.34, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 47.619, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d7b065b5aaa7e63e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 14978895, "90.00 percentile latency (ns)": 15701303, "90th percentile latency (ns)": 15701303, "95.00 percentile latency (ns)": 18743257, "97.00 percentile latency (ns)": 21746505, "99.00 percentile latency (ns)": 39244511, "99.90 percentile latency (ns)": 130168525, "Max latency (ns)": 163110827, "Mean latency (ns)": 16062937, "Min duration satisfied": "Yes", "Min latency (ns)": 14658530, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 62.19, "QPS w/o loadgen overhead": 62.26, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.668, "characteristics.good": 30834, "characteristics.power": 6.641900000000016, "characteristics.power.normalized_per_core": 0.055349166666666796, "characteristics.power.normalized_per_processor": 6.641900000000016, "characteristics.samples_per_second": 62.26, "characteristics.samples_per_second.normalized_per_core": 0.5188333333333334, "characteristics.samples_per_second.normalized_per_processor": 62.26, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 62.5, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "e8ef3c4ab4ad4933", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 26121054, "90.00 percentile latency (ns)": 28995621, "90th percentile latency (ns)": 28995621, "95.00 percentile latency (ns)": 35285101, "97.00 percentile latency (ns)": 41313167, "99.00 percentile latency (ns)": 83243512, "99.90 percentile latency (ns)": 102757433, "Max latency (ns)": 129375723, "Mean latency (ns)": 27798132, "Min duration satisfied": "Yes", "Min latency (ns)": 24852270, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 35.94, "QPS w/o loadgen overhead": 35.97, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.384, "characteristics.good": 33192, "characteristics.power": 6.904949999999988, "characteristics.power.normalized_per_core": 0.0575412499999999, "characteristics.power.normalized_per_processor": 6.904949999999988, "characteristics.samples_per_second": 35.97, "characteristics.samples_per_second.normalized_per_core": 0.29975, "characteristics.samples_per_second.normalized_per_processor": 35.97, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 34.4828, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "42c5a489a15fbb8d", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3644033, "90.00 percentile latency (ns)": 4250890, "90th percentile latency (ns)": 4250890, "95.00 percentile latency (ns)": 4484292, "97.00 percentile latency (ns)": 4691639, "99.00 percentile latency (ns)": 9725181, "99.90 percentile latency (ns)": 80592131, "Max latency (ns)": 144056503, "Mean latency (ns)": 4030395, "Min duration satisfied": "Yes", "Min latency (ns)": 3073730, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 245.3, "QPS w/o loadgen overhead": 248.11, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.11, "characteristics.good": 24555, "characteristics.power": 6.984399999999991, "characteristics.power.normalized_per_core": 0.05820333333333326, "characteristics.power.normalized_per_processor": 6.984399999999991, "characteristics.samples_per_second": 248.11, "characteristics.samples_per_second.normalized_per_core": 2.0675833333333333, "characteristics.samples_per_second.normalized_per_processor": 248.11, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "e400e8e508f84634", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7705480, "90.00 percentile latency (ns)": 8439592, "90th percentile latency (ns)": 8439592, "95.00 percentile latency (ns)": 8851510, "97.00 percentile latency (ns)": 11690486, "99.00 percentile latency (ns)": 23984859, "99.90 percentile latency (ns)": 119576659, "Max latency (ns)": 143654290, "Mean latency (ns)": 8361869, "Min duration satisfied": "Yes", "Min latency (ns)": 7137120, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 118.92, "QPS w/o loadgen overhead": 119.59, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 62.07, "characteristics.good": 31035, "characteristics.power": 7.134600000000001, "characteristics.power.normalized_per_core": 0.05945500000000001, "characteristics.power.normalized_per_processor": 7.134600000000001, "characteristics.samples_per_second": 119.59, "characteristics.samples_per_second.normalized_per_core": 0.9965833333333334, "characteristics.samples_per_second.normalized_per_processor": 119.59, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 111.111, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c188e7744864963a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 15962593, "90.00 percentile latency (ns)": 16541048, "90th percentile latency (ns)": 16541048, "95.00 percentile latency (ns)": 21031589, "97.00 percentile latency (ns)": 25709727, "99.00 percentile latency (ns)": 41201123, "99.90 percentile latency (ns)": 131935496, "Max latency (ns)": 153152332, "Mean latency (ns)": 17171174, "Min duration satisfied": "Yes", "Min latency (ns)": 15072238, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 58.18, "QPS w/o loadgen overhead": 58.24, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.224, "characteristics.good": 30612, "characteristics.power": 6.70453333333331, "characteristics.power.normalized_per_core": 0.05587111111111091, "characteristics.power.normalized_per_processor": 6.70453333333331, "characteristics.samples_per_second": 58.24, "characteristics.samples_per_second.normalized_per_core": 0.48533333333333334, "characteristics.samples_per_second.normalized_per_processor": 58.24, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 58.8235, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "466113564d1d4fba", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2093582, "90.00 percentile latency (ns)": 2776067, "90th percentile latency (ns)": 2776067, "95.00 percentile latency (ns)": 2969504, "97.00 percentile latency (ns)": 3078039, "99.00 percentile latency (ns)": 5575948, "99.90 percentile latency (ns)": 55956867, "Max latency (ns)": 82219182, "Mean latency (ns)": 2414338, "Min duration satisfied": "Yes", "Min latency (ns)": 1633962, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 408.3, "QPS w/o loadgen overhead": 414.19, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 42.384, "characteristics.good": 21192, "characteristics.power": 7.083833333333334, "characteristics.power.normalized_per_core": 0.05903194444444445, "characteristics.power.normalized_per_processor": 7.083833333333334, "characteristics.samples_per_second": 414.19, "characteristics.samples_per_second.normalized_per_core": 3.4515833333333332, "characteristics.samples_per_second.normalized_per_processor": 414.19, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 333.333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "bbfd048bd64fa8b7", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 84459898, "90.00 percentile latency (ns)": 100162077, "90th percentile latency (ns)": 100162077, "95.00 percentile latency (ns)": 105328694, "97.00 percentile latency (ns)": 110138384, "99.00 percentile latency (ns)": 197134373, "99.90 percentile latency (ns)": 223498136, "Max latency (ns)": 254658903, "Mean latency (ns)": 89179746, "Min duration satisfied": "Yes", "Min latency (ns)": 82869505, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 11.21, "QPS w/o loadgen overhead": 11.21, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.012, "characteristics.good": 37506, "characteristics.power": 6.833166666666666, "characteristics.power.normalized_per_core": 0.05694305555555555, "characteristics.power.normalized_per_processor": 6.833166666666666, "characteristics.samples_per_second": 11.21, "characteristics.samples_per_second.normalized_per_core": 0.09341666666666668, "characteristics.samples_per_second.normalized_per_processor": 11.21, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.3-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.3-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.3_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 11.6279, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "33346dea35d247fd", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10406777, "90.00 percentile latency (ns)": 11100370, "90th percentile latency (ns)": 11100370, "95.00 percentile latency (ns)": 12772587, "97.00 percentile latency (ns)": 15749132, "99.00 percentile latency (ns)": 34133582, "99.90 percentile latency (ns)": 88889609, "Max latency (ns)": 110195481, "Mean latency (ns)": 11231304, "Min duration satisfied": "Yes", "Min latency (ns)": 9789935, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 88.74, "QPS w/o loadgen overhead": 89.04, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.486, "characteristics.good": 32243, "characteristics.power": 7.167999999999993, "characteristics.power.normalized_per_core": 0.05973333333333328, "characteristics.power.normalized_per_processor": 7.167999999999993, "characteristics.samples_per_second": 89.04, "characteristics.samples_per_second.normalized_per_core": 0.7420000000000001, "characteristics.samples_per_second.normalized_per_processor": 89.04, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4b2e8be34326c26a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 131125188, "90.00 percentile latency (ns)": 148971978, "90th percentile latency (ns)": 148971978, "95.00 percentile latency (ns)": 151729123, "97.00 percentile latency (ns)": 153474424, "99.00 percentile latency (ns)": 163857698, "99.90 percentile latency (ns)": 275278770, "Max latency (ns)": 284203765, "Mean latency (ns)": 135080356, "Min duration satisfied": "Yes", "Min latency (ns)": 128946990, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.4, "QPS w/o loadgen overhead": 7.4, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 77.542, "characteristics.good": 38771, "characteristics.power": 7.004883333333358, "characteristics.power.normalized_per_core": 0.05837402777777798, "characteristics.power.normalized_per_processor": 7.004883333333358, "characteristics.samples_per_second": 7.4, "characteristics.samples_per_second.normalized_per_core": 0.06166666666666667, "characteristics.samples_per_second.normalized_per_processor": 7.4, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite2-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 7.35294, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d324b9184b3dc744", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4100408, "90.00 percentile latency (ns)": 4920391, "90th percentile latency (ns)": 4920391, "95.00 percentile latency (ns)": 5080646, "97.00 percentile latency (ns)": 5351955, "99.00 percentile latency (ns)": 13587624, "99.90 percentile latency (ns)": 63240987, "Max latency (ns)": 78657841, "Mean latency (ns)": 4597181, "Min duration satisfied": "Yes", "Min latency (ns)": 3661400, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 215.83, "QPS w/o loadgen overhead": 217.52, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.75, "characteristics.good": 28375, "characteristics.power": 7.167099999999984, "characteristics.power.normalized_per_core": 0.05972583333333319, "characteristics.power.normalized_per_processor": 7.167099999999984, "characteristics.samples_per_second": 217.52, "characteristics.samples_per_second.normalized_per_core": 1.8126666666666666, "characteristics.samples_per_second.normalized_per_processor": 217.52, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "049044edcb67dd91", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5291918, "90.00 percentile latency (ns)": 5870461, "90th percentile latency (ns)": 5870461, "95.00 percentile latency (ns)": 6176436, "97.00 percentile latency (ns)": 6885749, "99.00 percentile latency (ns)": 15932064, "99.90 percentile latency (ns)": 111159743, "Max latency (ns)": 133241757, "Mean latency (ns)": 5791754, "Min duration satisfied": "Yes", "Min latency (ns)": 4776109, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 171.31, "QPS w/o loadgen overhead": 172.66, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 57.002, "characteristics.good": 28501, "characteristics.power": 7.083333333333337, "characteristics.power.normalized_per_core": 0.059027777777777804, "characteristics.power.normalized_per_processor": 7.083333333333337, "characteristics.samples_per_second": 172.66, "characteristics.samples_per_second.normalized_per_core": 1.4388333333333334, "characteristics.samples_per_second.normalized_per_processor": 172.66, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "8083949258355755", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 90896977, "90.00 percentile latency (ns)": 107361194, "90th percentile latency (ns)": 107361194, "95.00 percentile latency (ns)": 112701786, "97.00 percentile latency (ns)": 118876780, "99.00 percentile latency (ns)": 201223641, "99.90 percentile latency (ns)": 229153486, "Max latency (ns)": 291484542, "Mean latency (ns)": 95689643, "Min duration satisfied": "Yes", "Min latency (ns)": 88728968, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 10.45, "QPS w/o loadgen overhead": 10.45, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.69, "characteristics.good": 37845, "characteristics.power": 6.907383333333331, "characteristics.power.normalized_per_core": 0.05756152777777776, "characteristics.power.normalized_per_processor": 6.907383333333331, "characteristics.samples_per_second": 10.45, "characteristics.samples_per_second.normalized_per_core": 0.08708333333333333, "characteristics.samples_per_second.normalized_per_processor": 10.45, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.4-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.4-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.4_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 10.6383, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "9deb2cdb41f48e68", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7068324, "90.00 percentile latency (ns)": 7958645, "90th percentile latency (ns)": 7958645, "95.00 percentile latency (ns)": 8328647, "97.00 percentile latency (ns)": 10416010, "99.00 percentile latency (ns)": 21582938, "99.90 percentile latency (ns)": 114972759, "Max latency (ns)": 171394386, "Mean latency (ns)": 7700767, "Min duration satisfied": "Yes", "Min latency (ns)": 6339989, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 128.91, "QPS w/o loadgen overhead": 129.86, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 52.446, "characteristics.good": 26223, "characteristics.power": 6.749133333333348, "characteristics.power.normalized_per_core": 0.0562427777777779, "characteristics.power.normalized_per_processor": 6.749133333333348, "characteristics.samples_per_second": 129.86, "characteristics.samples_per_second.normalized_per_core": 1.0821666666666667, "characteristics.samples_per_second.normalized_per_processor": 129.86, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "7aefcb6049546a6c", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 40118151, "90.00 percentile latency (ns)": 45209373, "90th percentile latency (ns)": 45209373, "95.00 percentile latency (ns)": 54823567, "97.00 percentile latency (ns)": 61175261, "99.00 percentile latency (ns)": 102917821, "99.90 percentile latency (ns)": 155126058, "Max latency (ns)": 171295609, "Mean latency (ns)": 42598893, "Min duration satisfied": "Yes", "Min latency (ns)": 39543146, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 23.46, "QPS w/o loadgen overhead": 23.47, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.724, "characteristics.good": 36862, "characteristics.power": 6.752216666666695, "characteristics.power.normalized_per_core": 0.056268472222222456, "characteristics.power.normalized_per_processor": 6.752216666666695, "characteristics.samples_per_second": 23.47, "characteristics.samples_per_second.normalized_per_core": 0.19558333333333333, "characteristics.samples_per_second.normalized_per_processor": 23.47, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 22.7273, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4d2f6210d4de2b72", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 30436141, "90.00 percentile latency (ns)": 34996891, "90th percentile latency (ns)": 34996891, "95.00 percentile latency (ns)": 43130179, "97.00 percentile latency (ns)": 50724949, "99.00 percentile latency (ns)": 91188134, "99.90 percentile latency (ns)": 113467761, "Max latency (ns)": 132999034, "Mean latency (ns)": 32546484, "Min duration satisfied": "Yes", "Min latency (ns)": 29064859, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 30.7, "QPS w/o loadgen overhead": 30.73, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.604, "characteristics.good": 33302, "characteristics.power": 6.79718333333335, "characteristics.power.normalized_per_core": 0.05664319444444458, "characteristics.power.normalized_per_processor": 6.79718333333335, "characteristics.samples_per_second": 30.73, "characteristics.samples_per_second.normalized_per_core": 0.25608333333333333, "characteristics.samples_per_second.normalized_per_processor": 30.73, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 30.303, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ce83346f4de29155", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 40736893, "90.00 percentile latency (ns)": 48067765, "90th percentile latency (ns)": 48067765, "95.00 percentile latency (ns)": 55459939, "97.00 percentile latency (ns)": 61748216, "99.00 percentile latency (ns)": 109298935, "99.90 percentile latency (ns)": 128867988, "Max latency (ns)": 140470186, "Mean latency (ns)": 43306714, "Min duration satisfied": "Yes", "Min latency (ns)": 39201922, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 23.08, "QPS w/o loadgen overhead": 23.09, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.106, "characteristics.good": 34553, "characteristics.power": 6.974816666666661, "characteristics.power.normalized_per_core": 0.058123472222222174, "characteristics.power.normalized_per_processor": 6.974816666666661, "characteristics.samples_per_second": 23.09, "characteristics.samples_per_second.normalized_per_core": 0.19241666666666665, "characteristics.samples_per_second.normalized_per_processor": 23.09, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.2558, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "a479883e787ef645", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11169553, "90.00 percentile latency (ns)": 11913943, "90th percentile latency (ns)": 11913943, "95.00 percentile latency (ns)": 14217550, "97.00 percentile latency (ns)": 17421370, "99.00 percentile latency (ns)": 33687538, "99.90 percentile latency (ns)": 125136697, "Max latency (ns)": 149942103, "Mean latency (ns)": 12081537, "Min duration satisfied": "Yes", "Min latency (ns)": 10390203, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 82.3, "QPS w/o loadgen overhead": 82.77, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.822, "characteristics.good": 29911, "characteristics.power": 6.782433333333354, "characteristics.power.normalized_per_core": 0.05652027777777795, "characteristics.power.normalized_per_processor": 6.782433333333354, "characteristics.samples_per_second": 82.77, "characteristics.samples_per_second.normalized_per_core": 0.68975, "characteristics.samples_per_second.normalized_per_processor": 82.77, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4b4ca51e0af7eff2", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20171082, "90.00 percentile latency (ns)": 20831399, "90th percentile latency (ns)": 20831399, "95.00 percentile latency (ns)": 24433557, "97.00 percentile latency (ns)": 28657972, "99.00 percentile latency (ns)": 46377639, "99.90 percentile latency (ns)": 101352084, "Max latency (ns)": 121799189, "Mean latency (ns)": 21339308, "Min duration satisfied": "Yes", "Min latency (ns)": 19541284, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 46.81, "QPS w/o loadgen overhead": 46.86, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.336, "characteristics.good": 34168, "characteristics.power": 6.519150000000034, "characteristics.power.normalized_per_core": 0.054326250000000284, "characteristics.power.normalized_per_processor": 6.519150000000034, "characteristics.samples_per_second": 46.86, "characteristics.samples_per_second.normalized_per_core": 0.3905, "characteristics.samples_per_second.normalized_per_processor": 46.86, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 45.4545, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "9406844c2c86c9f5", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 33786442, "90.00 percentile latency (ns)": 37775239, "90th percentile latency (ns)": 37775239, "95.00 percentile latency (ns)": 45162005, "97.00 percentile latency (ns)": 52736727, "99.00 percentile latency (ns)": 69749075, "99.90 percentile latency (ns)": 158692413, "Max latency (ns)": 171164330, "Mean latency (ns)": 36034103, "Min duration satisfied": "Yes", "Min latency (ns)": 33193956, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 27.72, "QPS w/o loadgen overhead": 27.75, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.834, "characteristics.good": 34417, "characteristics.power": 6.670016666666654, "characteristics.power.normalized_per_core": 0.05558347222222212, "characteristics.power.normalized_per_processor": 6.670016666666654, "characteristics.samples_per_second": 27.75, "characteristics.samples_per_second.normalized_per_core": 0.23125, "characteristics.samples_per_second.normalized_per_processor": 27.75, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 27.7778, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "b6266ac472e81038", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 31243755, "90.00 percentile latency (ns)": 32632430, "90th percentile latency (ns)": 32632430, "95.00 percentile latency (ns)": 42858433, "97.00 percentile latency (ns)": 49656051, "99.00 percentile latency (ns)": 55656361, "99.90 percentile latency (ns)": 80012654, "Max latency (ns)": 165114395, "Mean latency (ns)": 32597453, "Min duration satisfied": "Yes", "Min latency (ns)": 30387938, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 30.65, "QPS w/o loadgen overhead": 30.68, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.19, "characteristics.good": 35595, "characteristics.power": 7.459833333333301, "characteristics.power.normalized_per_core": 0.06216527777777751, "characteristics.power.normalized_per_processor": 7.459833333333301, "characteristics.samples_per_second": 30.68, "characteristics.samples_per_second.normalized_per_core": 0.25566666666666665, "characteristics.samples_per_second.normalized_per_processor": 30.68, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite0-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 29.4118, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "9bf81bc4c9895cc0", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 194885032, "90.00 percentile latency (ns)": 214691187, "90th percentile latency (ns)": 214691187, "95.00 percentile latency (ns)": 217338350, "97.00 percentile latency (ns)": 222163811, "99.00 percentile latency (ns)": 233445307, "99.90 percentile latency (ns)": 357887332, "Max latency (ns)": 375802409, "Mean latency (ns)": 200907135, "Min duration satisfied": "Yes", "Min latency (ns)": 192244108, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 4.98, "QPS w/o loadgen overhead": 4.98, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 79.268, "characteristics.good": 39634, "characteristics.power": 7.068766666666681, "characteristics.power.normalized_per_core": 0.05890638888888901, "characteristics.power.normalized_per_processor": 7.068766666666681, "characteristics.samples_per_second": 4.98, "characteristics.samples_per_second.normalized_per_core": 0.0415, "characteristics.samples_per_second.normalized_per_processor": 4.98, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite3-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 4.92611, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "737409e91bc450ab", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 17558859, "90.00 percentile latency (ns)": 18065896, "90th percentile latency (ns)": 18065896, "95.00 percentile latency (ns)": 20276276, "97.00 percentile latency (ns)": 23839880, "99.00 percentile latency (ns)": 42612084, "99.90 percentile latency (ns)": 97010787, "Max latency (ns)": 125975401, "Mean latency (ns)": 18565486, "Min duration satisfied": "Yes", "Min latency (ns)": 16938003, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 53.8, "QPS w/o loadgen overhead": 53.86, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.216, "characteristics.good": 33108, "characteristics.power": 6.483166666666645, "characteristics.power.normalized_per_core": 0.05402638888888871, "characteristics.power.normalized_per_processor": 6.483166666666645, "characteristics.samples_per_second": 53.86, "characteristics.samples_per_second.normalized_per_core": 0.4488333333333333, "characteristics.samples_per_second.normalized_per_processor": 53.86, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 52.6316, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d43e5bd5840906e3", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 95705644, "90.00 percentile latency (ns)": 114004448, "90th percentile latency (ns)": 114004448, "95.00 percentile latency (ns)": 117178218, "97.00 percentile latency (ns)": 118700427, "99.00 percentile latency (ns)": 126629390, "99.90 percentile latency (ns)": 223167413, "Max latency (ns)": 259321589, "Mean latency (ns)": 99061728, "Min duration satisfied": "Yes", "Min latency (ns)": 94019889, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 10.09, "QPS w/o loadgen overhead": 10.09, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.344, "characteristics.good": 38172, "characteristics.power": 7.513483333333378, "characteristics.power.normalized_per_core": 0.06261236111111149, "characteristics.power.normalized_per_processor": 7.513483333333378, "characteristics.samples_per_second": 10.09, "characteristics.samples_per_second.normalized_per_core": 0.08408333333333333, "characteristics.samples_per_second.normalized_per_processor": 10.09, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "efficientnet-lite3-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 9.90099, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "34fa622ce8c8b695", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5428940, "90.00 percentile latency (ns)": 5673163, "90th percentile latency (ns)": 5673163, "95.00 percentile latency (ns)": 5837948, "97.00 percentile latency (ns)": 7142195, "99.00 percentile latency (ns)": 14816343, "99.90 percentile latency (ns)": 106243970, "Max latency (ns)": 149225353, "Mean latency (ns)": 5806551, "Min duration satisfied": "Yes", "Min latency (ns)": 4994689, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 171.59, "QPS w/o loadgen overhead": 172.22, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 54.568, "characteristics.good": 27284, "characteristics.power": 7.074166666666674, "characteristics.power.normalized_per_core": 0.05895138888888895, "characteristics.power.normalized_per_processor": 7.074166666666674, "characteristics.samples_per_second": 172.22, "characteristics.samples_per_second.normalized_per_core": 1.4351666666666667, "characteristics.samples_per_second.normalized_per_processor": 172.22, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c08665861495aa9d", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9581250, "90.00 percentile latency (ns)": 9899724, "90th percentile latency (ns)": 9899724, "95.00 percentile latency (ns)": 10245564, "97.00 percentile latency (ns)": 11880315, "99.00 percentile latency (ns)": 32773540, "99.90 percentile latency (ns)": 71669844, "Max latency (ns)": 111685506, "Mean latency (ns)": 10215090, "Min duration satisfied": "Yes", "Min latency (ns)": 9221951, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 97.72, "QPS w/o loadgen overhead": 97.89, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.478, "characteristics.good": 24739, "characteristics.power": 6.419333333333331, "characteristics.power.normalized_per_core": 0.05349444444444442, "characteristics.power.normalized_per_processor": 6.419333333333331, "characteristics.samples_per_second": 97.89, "characteristics.samples_per_second.normalized_per_core": 0.81575, "characteristics.samples_per_second.normalized_per_processor": 97.89, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "14cfe973fdbd10bf", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 18186561, "90.00 percentile latency (ns)": 19170551, "90th percentile latency (ns)": 19170551, "95.00 percentile latency (ns)": 25110560, "97.00 percentile latency (ns)": 30868884, "99.00 percentile latency (ns)": 44720947, "99.90 percentile latency (ns)": 95442005, "Max latency (ns)": 112225732, "Mean latency (ns)": 19555222, "Min duration satisfied": "Yes", "Min latency (ns)": 17473130, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 51.02, "QPS w/o loadgen overhead": 51.14, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.126, "characteristics.good": 34063, "characteristics.power": 7.334983333333313, "characteristics.power.normalized_per_core": 0.06112486111111094, "characteristics.power.normalized_per_processor": 7.334983333333313, "characteristics.samples_per_second": 51.14, "characteristics.samples_per_second.normalized_per_core": 0.4261666666666667, "characteristics.samples_per_second.normalized_per_processor": 51.14, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "9a41a241d49039f1", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 49941974, "90.00 percentile latency (ns)": 58012647, "90th percentile latency (ns)": 58012647, "95.00 percentile latency (ns)": 68523288, "97.00 percentile latency (ns)": 72332722, "99.00 percentile latency (ns)": 126320661, "99.90 percentile latency (ns)": 172375466, "Max latency (ns)": 186073332, "Mean latency (ns)": 52911400, "Min duration satisfied": "Yes", "Min latency (ns)": 48680427, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 18.89, "QPS w/o loadgen overhead": 18.9, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.76, "characteristics.good": 37880, "characteristics.power": 6.814566666666678, "characteristics.power.normalized_per_core": 0.056788055555555654, "characteristics.power.normalized_per_processor": 6.814566666666678, "characteristics.samples_per_second": 18.9, "characteristics.samples_per_second.normalized_per_core": 0.1575, "characteristics.samples_per_second.normalized_per_processor": 18.9, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.8679, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "000c753b4cc599a9", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 28360728, "90.00 percentile latency (ns)": 30144317, "90th percentile latency (ns)": 30144317, "95.00 percentile latency (ns)": 37085774, "97.00 percentile latency (ns)": 44859676, "99.00 percentile latency (ns)": 57423473, "99.90 percentile latency (ns)": 148850934, "Max latency (ns)": 221463461, "Mean latency (ns)": 30187741, "Min duration satisfied": "Yes", "Min latency (ns)": 27848811, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 33.08, "QPS w/o loadgen overhead": 33.13, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 65.75, "characteristics.good": 32875, "characteristics.power": 6.459899999999988, "characteristics.power.normalized_per_core": 0.0538324999999999, "characteristics.power.normalized_per_processor": 6.459899999999988, "characteristics.samples_per_second": 33.13, "characteristics.samples_per_second.normalized_per_core": 0.27608333333333335, "characteristics.samples_per_second.normalized_per_processor": 33.13, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 33.3333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "22574fcb80c1e5a4", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10520224, "90.00 percentile latency (ns)": 11300137, "90th percentile latency (ns)": 11300137, "95.00 percentile latency (ns)": 14050039, "97.00 percentile latency (ns)": 16852219, "99.00 percentile latency (ns)": 34060704, "99.90 percentile latency (ns)": 78536010, "Max latency (ns)": 119194809, "Mean latency (ns)": 11438954, "Min duration satisfied": "Yes", "Min latency (ns)": 9978527, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 87.16, "QPS w/o loadgen overhead": 87.42, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.768, "characteristics.good": 32384, "characteristics.power": 7.255133333333362, "characteristics.power.normalized_per_core": 0.060459444444444684, "characteristics.power.normalized_per_processor": 7.255133333333362, "characteristics.samples_per_second": 87.42, "characteristics.samples_per_second.normalized_per_core": 0.7285, "characteristics.samples_per_second.normalized_per_processor": 87.42, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "43b5aa48cb78d777", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 45667385, "90.00 percentile latency (ns)": 51733868, "90th percentile latency (ns)": 51733868, "95.00 percentile latency (ns)": 61864988, "97.00 percentile latency (ns)": 66155137, "99.00 percentile latency (ns)": 131656764, "99.90 percentile latency (ns)": 169670518, "Max latency (ns)": 217990263, "Mean latency (ns)": 48405601, "Min duration satisfied": "Yes", "Min latency (ns)": 44853281, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.65, "QPS w/o loadgen overhead": 20.66, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.122, "characteristics.good": 35061, "characteristics.power": 6.64043333333334, "characteristics.power.normalized_per_core": 0.0553369444444445, "characteristics.power.normalized_per_processor": 6.64043333333334, "characteristics.samples_per_second": 20.66, "characteristics.samples_per_second.normalized_per_core": 0.17216666666666666, "characteristics.samples_per_second.normalized_per_processor": 20.66, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 20.8333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "fe927199ce8dc391", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10200883, "90.00 percentile latency (ns)": 10921819, "90th percentile latency (ns)": 10921819, "95.00 percentile latency (ns)": 13293539, "97.00 percentile latency (ns)": 16741277, "99.00 percentile latency (ns)": 34202905, "99.90 percentile latency (ns)": 78824574, "Max latency (ns)": 121137809, "Mean latency (ns)": 11094742, "Min duration satisfied": "Yes", "Min latency (ns)": 9619988, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 89.88, "QPS w/o loadgen overhead": 90.13, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.08, "characteristics.good": 32040, "characteristics.power": 7.263033333333353, "characteristics.power.normalized_per_core": 0.06052527777777794, "characteristics.power.normalized_per_processor": 7.263033333333353, "characteristics.samples_per_second": 90.13, "characteristics.samples_per_second.normalized_per_core": 0.7510833333333333, "characteristics.samples_per_second.normalized_per_processor": 90.13, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "3dca0f0151669812", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 21072097, "90.00 percentile latency (ns)": 22084574, "90th percentile latency (ns)": 22084574, "95.00 percentile latency (ns)": 25829400, "97.00 percentile latency (ns)": 31140119, "99.00 percentile latency (ns)": 47024383, "99.90 percentile latency (ns)": 139799599, "Max latency (ns)": 227827422, "Mean latency (ns)": 22479101, "Min duration satisfied": "Yes", "Min latency (ns)": 20658736, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 44.45, "QPS w/o loadgen overhead": 44.49, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.282, "characteristics.good": 32141, "characteristics.power": 6.5180666666667015, "characteristics.power.normalized_per_core": 0.05431722222222251, "characteristics.power.normalized_per_processor": 6.5180666666667015, "characteristics.samples_per_second": 44.49, "characteristics.samples_per_second.normalized_per_core": 0.37075, "characteristics.samples_per_second.normalized_per_processor": 44.49, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 45.4545, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "1adb033649f24070", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 41081995, "90.00 percentile latency (ns)": 46306662, "90th percentile latency (ns)": 46306662, "95.00 percentile latency (ns)": 56187405, "97.00 percentile latency (ns)": 62860688, "99.00 percentile latency (ns)": 97326510, "99.90 percentile latency (ns)": 157926923, "Max latency (ns)": 170274715, "Mean latency (ns)": 43605545, "Min duration satisfied": "Yes", "Min latency (ns)": 40288440, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 22.92, "QPS w/o loadgen overhead": 22.93, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.678, "characteristics.good": 36339, "characteristics.power": 6.714083333333302, "characteristics.power.normalized_per_core": 0.055950694444444185, "characteristics.power.normalized_per_processor": 6.714083333333302, "characteristics.samples_per_second": 22.93, "characteristics.samples_per_second.normalized_per_core": 0.19108333333333333, "characteristics.samples_per_second.normalized_per_processor": 22.93, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 22.2222, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "42d1ee4979956b5b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11350324, "90.00 percentile latency (ns)": 11610320, "90th percentile latency (ns)": 11610320, "95.00 percentile latency (ns)": 14604887, "97.00 percentile latency (ns)": 17457191, "99.00 percentile latency (ns)": 34835139, "99.90 percentile latency (ns)": 116251904, "Max latency (ns)": 204371131, "Mean latency (ns)": 12144457, "Min duration satisfied": "Yes", "Min latency (ns)": 10668918, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 82.22, "QPS w/o loadgen overhead": 82.34, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.972, "characteristics.good": 32486, "characteristics.power": 7.1893666666666425, "characteristics.power.normalized_per_core": 0.05991138888888869, "characteristics.power.normalized_per_processor": 7.1893666666666425, "characteristics.samples_per_second": 82.34, "characteristics.samples_per_second.normalized_per_core": 0.6861666666666667, "characteristics.samples_per_second.normalized_per_processor": 82.34, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "fa6e5e41d7a30e99", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 57094427, "90.00 percentile latency (ns)": 67587200, "90th percentile latency (ns)": 67587200, "95.00 percentile latency (ns)": 76605658, "97.00 percentile latency (ns)": 80820738, "99.00 percentile latency (ns)": 163746088, "99.90 percentile latency (ns)": 187468321, "Max latency (ns)": 200837163, "Mean latency (ns)": 60624808, "Min duration satisfied": "Yes", "Min latency (ns)": 55835571, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.49, "QPS w/o loadgen overhead": 16.49, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.77, "characteristics.good": 35885, "characteristics.power": 6.730683333333318, "characteristics.power.normalized_per_core": 0.056089027777777654, "characteristics.power.normalized_per_processor": 6.730683333333318, "characteristics.samples_per_second": 16.49, "characteristics.samples_per_second.normalized_per_core": 0.13741666666666666, "characteristics.samples_per_second.normalized_per_processor": 16.49, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 16.9492, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d59998e7e6541b7b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 24660909, "90.00 percentile latency (ns)": 26152278, "90th percentile latency (ns)": 26152278, "95.00 percentile latency (ns)": 32759809, "97.00 percentile latency (ns)": 37767319, "99.00 percentile latency (ns)": 51168529, "99.90 percentile latency (ns)": 143532155, "Max latency (ns)": 185307314, "Mean latency (ns)": 26226806, "Min duration satisfied": "Yes", "Min latency (ns)": 23824070, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 38.11, "QPS w/o loadgen overhead": 38.13, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.75, "characteristics.good": 33375, "characteristics.power": 6.701983333333306, "characteristics.power.normalized_per_core": 0.055849861111110885, "characteristics.power.normalized_per_processor": 6.701983333333306, "characteristics.samples_per_second": 38.13, "characteristics.samples_per_second.normalized_per_core": 0.31775000000000003, "characteristics.samples_per_second.normalized_per_processor": 38.13, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 38.4615, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ae4017838b91b8a1", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5677428, "90.00 percentile latency (ns)": 6278508, "90th percentile latency (ns)": 6278508, "95.00 percentile latency (ns)": 6644666, "97.00 percentile latency (ns)": 7868139, "99.00 percentile latency (ns)": 17144968, "99.90 percentile latency (ns)": 97800628, "Max latency (ns)": 117593777, "Mean latency (ns)": 6176324, "Min duration satisfied": "Yes", "Min latency (ns)": 5162681, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 160.38, "QPS w/o loadgen overhead": 161.91, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 47.14, "characteristics.good": 23570, "characteristics.power": 6.709999999999976, "characteristics.power.normalized_per_core": 0.05591666666666647, "characteristics.power.normalized_per_processor": 6.709999999999976, "characteristics.samples_per_second": 161.91, "characteristics.samples_per_second.normalized_per_core": 1.34925, "characteristics.samples_per_second.normalized_per_processor": 161.91, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "307524f62b64b36e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11985406, "90.00 percentile latency (ns)": 12619115, "90th percentile latency (ns)": 12619115, "95.00 percentile latency (ns)": 14531642, "97.00 percentile latency (ns)": 16826811, "99.00 percentile latency (ns)": 32102162, "99.90 percentile latency (ns)": 125998623, "Max latency (ns)": 177156942, "Mean latency (ns)": 12863058, "Min duration satisfied": "Yes", "Min latency (ns)": 11667723, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 77.65, "QPS w/o loadgen overhead": 77.74, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.988, "characteristics.good": 28494, "characteristics.power": 6.496049999999984, "characteristics.power.normalized_per_core": 0.05413374999999987, "characteristics.power.normalized_per_processor": 6.496049999999984, "characteristics.samples_per_second": 77.74, "characteristics.samples_per_second.normalized_per_core": 0.6478333333333333, "characteristics.samples_per_second.normalized_per_processor": 77.74, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 76.9231, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "e55a8962441345e8", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 18806541, "90.00 percentile latency (ns)": 20017454, "90th percentile latency (ns)": 20017454, "95.00 percentile latency (ns)": 26017838, "97.00 percentile latency (ns)": 32689003, "99.00 percentile latency (ns)": 48446341, "99.90 percentile latency (ns)": 92389931, "Max latency (ns)": 105565960, "Mean latency (ns)": 20231230, "Min duration satisfied": "Yes", "Min latency (ns)": 17962506, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 49.29, "QPS w/o loadgen overhead": 49.43, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.716, "characteristics.good": 31858, "characteristics.power": 6.795733333333348, "characteristics.power.normalized_per_core": 0.05663111111111123, "characteristics.power.normalized_per_processor": 6.795733333333348, "characteristics.samples_per_second": 49.43, "characteristics.samples_per_second.normalized_per_core": 0.41191666666666665, "characteristics.samples_per_second.normalized_per_processor": 49.43, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 47.619, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6a501ab2f4f0aa42", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5031137, "90.00 percentile latency (ns)": 5700043, "90th percentile latency (ns)": 5700043, "95.00 percentile latency (ns)": 6002672, "97.00 percentile latency (ns)": 6615469, "99.00 percentile latency (ns)": 14562004, "99.90 percentile latency (ns)": 104705529, "Max latency (ns)": 150546800, "Mean latency (ns)": 5511537, "Min duration satisfied": "Yes", "Min latency (ns)": 4500721, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 179.45, "QPS w/o loadgen overhead": 181.44, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 55.54, "characteristics.good": 27770, "characteristics.power": 7.050000000000009, "characteristics.power.normalized_per_core": 0.05875000000000007, "characteristics.power.normalized_per_processor": 7.050000000000009, "characteristics.samples_per_second": 181.44, "characteristics.samples_per_second.normalized_per_core": 1.512, "characteristics.samples_per_second.normalized_per_processor": 181.44, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "3140dac4020b1760", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 16936760, "90.00 percentile latency (ns)": 17751669, "90th percentile latency (ns)": 17751669, "95.00 percentile latency (ns)": 19628879, "97.00 percentile latency (ns)": 22673302, "99.00 percentile latency (ns)": 41517817, "99.90 percentile latency (ns)": 131144415, "Max latency (ns)": 156997688, "Mean latency (ns)": 17962880, "Min duration satisfied": "Yes", "Min latency (ns)": 16243423, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 55.57, "QPS w/o loadgen overhead": 55.67, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.322, "characteristics.good": 29661, "characteristics.power": 6.415633333333327, "characteristics.power.normalized_per_core": 0.05346361111111106, "characteristics.power.normalized_per_processor": 6.415633333333327, "characteristics.samples_per_second": 55.67, "characteristics.samples_per_second.normalized_per_core": 0.4639166666666667, "characteristics.samples_per_second.normalized_per_processor": 55.67, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 58.8235, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "411504946769847a", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 30763201, "90.00 percentile latency (ns)": 114446581, "90th percentile latency (ns)": 114446581, "95.00 percentile latency (ns)": 123667394, "97.00 percentile latency (ns)": 127334940, "99.00 percentile latency (ns)": 133064528, "99.90 percentile latency (ns)": 142796268, "Max latency (ns)": 149314034, "Mean latency (ns)": 48152423, "Min duration satisfied": "Yes", "Min latency (ns)": 29080283, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.74, "QPS w/o loadgen overhead": 20.77, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.242, "characteristics.good": 34121, "characteristics.samples_per_second": 20.77, "characteristics.samples_per_second.normalized_per_core": 0.17308333333333334, "characteristics.samples_per_second.normalized_per_processor": 20.77, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.5185, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "044e2abc53b91bc4", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 42127216, "90.00 percentile latency (ns)": 173981023, "90th percentile latency (ns)": 173981023, "95.00 percentile latency (ns)": 199796761, "97.00 percentile latency (ns)": 206145968, "99.00 percentile latency (ns)": 213969411, "99.90 percentile latency (ns)": 226714874, "Max latency (ns)": 238719068, "Mean latency (ns)": 67924614, "Min duration satisfied": "Yes", "Min latency (ns)": 37262506, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 14.71, "QPS w/o loadgen overhead": 14.72, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.738, "characteristics.good": 35369, "characteristics.samples_per_second": 14.72, "characteristics.samples_per_second.normalized_per_core": 0.12266666666666667, "characteristics.samples_per_second.normalized_per_processor": 14.72, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.2857, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "744267843c42edbd", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 346502558, "90.00 percentile latency (ns)": 424471172, "90th percentile latency (ns)": 424471172, "95.00 percentile latency (ns)": 436307405, "97.00 percentile latency (ns)": 442650029, "99.00 percentile latency (ns)": 457533679, "99.90 percentile latency (ns)": 491218165, "Max latency (ns)": 509277804, "Mean latency (ns)": 321671767, "Min duration satisfied": "Yes", "Min latency (ns)": 177371225, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 3.11, "QPS w/o loadgen overhead": 3.11, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 78.514, "characteristics.good": 39257, "characteristics.samples_per_second": 3.11, "characteristics.samples_per_second.normalized_per_core": 0.025916666666666664, "characteristics.samples_per_second.normalized_per_processor": 3.11, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite4-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.3557, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "136b549afb0a1eca", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 13487226, "90.00 percentile latency (ns)": 22662759, "90th percentile latency (ns)": 22662759, "95.00 percentile latency (ns)": 45350185, "97.00 percentile latency (ns)": 94955439, "99.00 percentile latency (ns)": 167991168, "99.90 percentile latency (ns)": 190578391, "Max latency (ns)": 209101540, "Mean latency (ns)": 19805001, "Min duration satisfied": "Yes", "Min latency (ns)": 12924315, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 50.35, "QPS w/o loadgen overhead": 50.49, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.542, "characteristics.good": 31771, "characteristics.samples_per_second": 50.49, "characteristics.samples_per_second.normalized_per_core": 0.42075, "characteristics.samples_per_second.normalized_per_processor": 50.49, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 38.4615, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d695bcc5a69c435d", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10245367, "90.00 percentile latency (ns)": 109642952, "90th percentile latency (ns)": 109642952, "95.00 percentile latency (ns)": 155122254, "97.00 percentile latency (ns)": 165128160, "99.00 percentile latency (ns)": 175027872, "99.90 percentile latency (ns)": 189830877, "Max latency (ns)": 207258244, "Mean latency (ns)": 30800342, "Min duration satisfied": "Yes", "Min latency (ns)": 8441996, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 32.42, "QPS w/o loadgen overhead": 32.47, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 52.172, "characteristics.good": 26086, "characteristics.samples_per_second": 32.47, "characteristics.samples_per_second.normalized_per_core": 0.27058333333333334, "characteristics.samples_per_second.normalized_per_processor": 32.47, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 38.4615, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ebc7b3bd4b28a1e0", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 18168184, "90.00 percentile latency (ns)": 165790062, "90th percentile latency (ns)": 165790062, "95.00 percentile latency (ns)": 175324384, "97.00 percentile latency (ns)": 179661044, "99.00 percentile latency (ns)": 187219857, "99.90 percentile latency (ns)": 200246211, "Max latency (ns)": 214645620, "Mean latency (ns)": 55267573, "Min duration satisfied": "Yes", "Min latency (ns)": 10581360, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 18.08, "QPS w/o loadgen overhead": 18.09, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.402, "characteristics.good": 29201, "characteristics.samples_per_second": 18.09, "characteristics.samples_per_second.normalized_per_core": 0.15075, "characteristics.samples_per_second.normalized_per_processor": 18.09, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 16.3934, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ff487a3852243759", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20934920, "90.00 percentile latency (ns)": 160555092, "90th percentile latency (ns)": 160555092, "95.00 percentile latency (ns)": 183010100, "97.00 percentile latency (ns)": 187815509, "99.00 percentile latency (ns)": 194947194, "99.90 percentile latency (ns)": 207713795, "Max latency (ns)": 217032535, "Mean latency (ns)": 48626198, "Min duration satisfied": "Yes", "Min latency (ns)": 19731778, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.55, "QPS w/o loadgen overhead": 20.57, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 60.976, "characteristics.good": 30488, "characteristics.samples_per_second": 20.57, "characteristics.samples_per_second.normalized_per_core": 0.17141666666666666, "characteristics.samples_per_second.normalized_per_processor": 20.57, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.873, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "37448840468993ae", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4978728, "90.00 percentile latency (ns)": 5449470, "90th percentile latency (ns)": 5449470, "95.00 percentile latency (ns)": 6504422, "97.00 percentile latency (ns)": 9158598, "99.00 percentile latency (ns)": 62129678, "99.90 percentile latency (ns)": 98663527, "Max latency (ns)": 117833657, "Mean latency (ns)": 6111088, "Min duration satisfied": "Yes", "Min latency (ns)": 3911819, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 162.89, "QPS w/o loadgen overhead": 163.64, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 48.162, "characteristics.good": 24081, "characteristics.samples_per_second": 163.64, "characteristics.samples_per_second.normalized_per_core": 1.3636666666666666, "characteristics.samples_per_second.normalized_per_processor": 163.64, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d2d48f78b7219758", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 75170822, "90.00 percentile latency (ns)": 173412730, "90th percentile latency (ns)": 173412730, "95.00 percentile latency (ns)": 201737889, "97.00 percentile latency (ns)": 208395711, "99.00 percentile latency (ns)": 216914774, "99.90 percentile latency (ns)": 232927248, "Max latency (ns)": 246852848, "Mean latency (ns)": 91646769, "Min duration satisfied": "Yes", "Min latency (ns)": 63542968, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 10.91, "QPS w/o loadgen overhead": 10.91, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.068, "characteristics.good": 37534, "characteristics.samples_per_second": 10.91, "characteristics.samples_per_second.normalized_per_core": 0.09091666666666667, "characteristics.samples_per_second.normalized_per_processor": 10.91, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite0-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 10.8696, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "3ebb4d89f8a0a5e8", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5547229, "90.00 percentile latency (ns)": 7076455, "90th percentile latency (ns)": 7076455, "95.00 percentile latency (ns)": 15760811, "97.00 percentile latency (ns)": 42230634, "99.00 percentile latency (ns)": 90812168, "99.90 percentile latency (ns)": 102818350, "Max latency (ns)": 122846429, "Mean latency (ns)": 8415294, "Min duration satisfied": "Yes", "Min latency (ns)": 5039733, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 118.12, "QPS w/o loadgen overhead": 118.83, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 50.242, "characteristics.good": 25121, "characteristics.samples_per_second": 118.83, "characteristics.samples_per_second.normalized_per_core": 0.99025, "characteristics.samples_per_second.normalized_per_processor": 118.83, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "8b6652b480483e42", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 21354090, "90.00 percentile latency (ns)": 104244588, "90th percentile latency (ns)": 104244588, "95.00 percentile latency (ns)": 107854282, "97.00 percentile latency (ns)": 109845490, "99.00 percentile latency (ns)": 113643298, "99.90 percentile latency (ns)": 121020413, "Max latency (ns)": 127683805, "Mean latency (ns)": 46115094, "Min duration satisfied": "Yes", "Min latency (ns)": 12918506, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 21.65, "QPS w/o loadgen overhead": 21.68, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 60.842, "characteristics.good": 30421, "characteristics.samples_per_second": 21.68, "characteristics.samples_per_second.normalized_per_core": 0.18066666666666667, "characteristics.samples_per_second.normalized_per_processor": 21.68, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.8679, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "99a8b1e73645e183", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 29894471, "90.00 percentile latency (ns)": 152025071, "90th percentile latency (ns)": 152025071, "95.00 percentile latency (ns)": 189338427, "97.00 percentile latency (ns)": 196632043, "99.00 percentile latency (ns)": 205507546, "99.90 percentile latency (ns)": 217165299, "Max latency (ns)": 230662255, "Mean latency (ns)": 54557913, "Min duration satisfied": "Yes", "Min latency (ns)": 28489104, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 18.31, "QPS w/o loadgen overhead": 18.33, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.016, "characteristics.good": 34508, "characteristics.samples_per_second": 18.33, "characteristics.samples_per_second.normalized_per_core": 0.15275, "characteristics.samples_per_second.normalized_per_processor": 18.33, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 20.4082, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c1a211442acac089", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 58578702, "90.00 percentile latency (ns)": 137455856, "90th percentile latency (ns)": 137455856, "95.00 percentile latency (ns)": 204457149, "97.00 percentile latency (ns)": 231053265, "99.00 percentile latency (ns)": 252404368, "99.90 percentile latency (ns)": 268565715, "Max latency (ns)": 281055103, "Mean latency (ns)": 76987388, "Min duration satisfied": "Yes", "Min latency (ns)": 53485611, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 12.99, "QPS w/o loadgen overhead": 12.99, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.098, "characteristics.good": 37049, "characteristics.samples_per_second": 12.99, "characteristics.samples_per_second.normalized_per_core": 0.10825, "characteristics.samples_per_second.normalized_per_processor": 12.99, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite1-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.987, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "0817d4085b0cb554", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11471829, "90.00 percentile latency (ns)": 14024784, "90th percentile latency (ns)": 14024784, "95.00 percentile latency (ns)": 25236449, "97.00 percentile latency (ns)": 40427025, "99.00 percentile latency (ns)": 137162308, "99.90 percentile latency (ns)": 186633923, "Max latency (ns)": 206164996, "Mean latency (ns)": 14801132, "Min duration satisfied": "Yes", "Min latency (ns)": 10044996, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 67.37, "QPS w/o loadgen overhead": 67.56, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.924, "characteristics.good": 30962, "characteristics.samples_per_second": 67.56, "characteristics.samples_per_second.normalized_per_core": 0.5630000000000001, "characteristics.samples_per_second.normalized_per_processor": 67.56, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "e1a3563cc8422ca8", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11032608, "90.00 percentile latency (ns)": 19939450, "90th percentile latency (ns)": 19939450, "95.00 percentile latency (ns)": 59852761, "97.00 percentile latency (ns)": 91688576, "99.00 percentile latency (ns)": 106028339, "99.90 percentile latency (ns)": 116041422, "Max latency (ns)": 125118776, "Mean latency (ns)": 16536215, "Min duration satisfied": "Yes", "Min latency (ns)": 10332614, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 60.26, "QPS w/o loadgen overhead": 60.47, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 62.298, "characteristics.good": 31149, "characteristics.samples_per_second": 60.47, "characteristics.samples_per_second.normalized_per_core": 0.5039166666666667, "characteristics.samples_per_second.normalized_per_processor": 60.47, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 62.5, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "bf05048798530afd", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5232501, "90.00 percentile latency (ns)": 5762459, "90th percentile latency (ns)": 5762459, "95.00 percentile latency (ns)": 7194251, "97.00 percentile latency (ns)": 8306668, "99.00 percentile latency (ns)": 37949630, "99.90 percentile latency (ns)": 163418483, "Max latency (ns)": 198244697, "Mean latency (ns)": 6411965, "Min duration satisfied": "Yes", "Min latency (ns)": 4496626, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 155.12, "QPS w/o loadgen overhead": 155.96, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.12, "characteristics.good": 24560, "characteristics.samples_per_second": 155.96, "characteristics.samples_per_second.normalized_per_core": 1.2996666666666667, "characteristics.samples_per_second.normalized_per_processor": 155.96, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "7e52fccb87d4275b", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 41458361, "90.00 percentile latency (ns)": 79052623, "90th percentile latency (ns)": 79052623, "95.00 percentile latency (ns)": 159048522, "97.00 percentile latency (ns)": 189535308, "99.00 percentile latency (ns)": 218413011, "99.90 percentile latency (ns)": 238339032, "Max latency (ns)": 247931861, "Mean latency (ns)": 55819472, "Min duration satisfied": "Yes", "Min latency (ns)": 39703533, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.9, "QPS w/o loadgen overhead": 17.91, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.584, "characteristics.good": 36792, "characteristics.samples_per_second": 17.91, "characteristics.samples_per_second.normalized_per_core": 0.14925, "characteristics.samples_per_second.normalized_per_processor": 17.91, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.4-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.4-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_140.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.1818, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "489c042048e4b6ab", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 29576340, "90.00 percentile latency (ns)": 60099600, "90th percentile latency (ns)": 60099600, "95.00 percentile latency (ns)": 107319235, "97.00 percentile latency (ns)": 120360805, "99.00 percentile latency (ns)": 129164664, "99.90 percentile latency (ns)": 138505799, "Max latency (ns)": 147766787, "Mean latency (ns)": 38860040, "Min duration satisfied": "Yes", "Min latency (ns)": 28503990, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 25.69, "QPS w/o loadgen overhead": 25.73, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.772, "characteristics.good": 35386, "characteristics.samples_per_second": 25.73, "characteristics.samples_per_second.normalized_per_core": 0.21441666666666667, "characteristics.samples_per_second.normalized_per_processor": 25.73, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 24.3902, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "5d2e490e8f1507d7", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10146803, "90.00 percentile latency (ns)": 84282123, "90th percentile latency (ns)": 84282123, "95.00 percentile latency (ns)": 90882367, "97.00 percentile latency (ns)": 94233491, "99.00 percentile latency (ns)": 99388810, "99.90 percentile latency (ns)": 108333830, "Max latency (ns)": 118658591, "Mean latency (ns)": 27389143, "Min duration satisfied": "Yes", "Min latency (ns)": 6490766, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 36.42, "QPS w/o loadgen overhead": 36.51, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 47.808, "characteristics.good": 23904, "characteristics.samples_per_second": 36.51, "characteristics.samples_per_second.normalized_per_core": 0.30424999999999996, "characteristics.samples_per_second.normalized_per_processor": 36.51, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 37.037, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "f92442ec92b3f265", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 28018971, "90.00 percentile latency (ns)": 174878028, "90th percentile latency (ns)": 174878028, "95.00 percentile latency (ns)": 183326114, "97.00 percentile latency (ns)": 187289041, "99.00 percentile latency (ns)": 193774779, "99.90 percentile latency (ns)": 202575822, "Max latency (ns)": 211701953, "Mean latency (ns)": 64713813, "Min duration satisfied": "Yes", "Min latency (ns)": 16156007, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 15.45, "QPS w/o loadgen overhead": 15.45, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.038, "characteristics.good": 32019, "characteristics.samples_per_second": 15.45, "characteristics.samples_per_second.normalized_per_core": 0.12875, "characteristics.samples_per_second.normalized_per_processor": 15.45, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.5, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4aa8b07c5f7609dc", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 13911616, "90.00 percentile latency (ns)": 92078234, "90th percentile latency (ns)": 92078234, "95.00 percentile latency (ns)": 97996148, "97.00 percentile latency (ns)": 100854522, "99.00 percentile latency (ns)": 106075310, "99.90 percentile latency (ns)": 115182054, "Max latency (ns)": 138363415, "Mean latency (ns)": 32087241, "Min duration satisfied": "Yes", "Min latency (ns)": 11596077, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 31.12, "QPS w/o loadgen overhead": 31.17, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 51.848, "characteristics.good": 25924, "characteristics.samples_per_second": 31.17, "characteristics.samples_per_second.normalized_per_core": 0.25975000000000004, "characteristics.samples_per_second.normalized_per_processor": 31.17, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 21.2766, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "c05b4b483badb8c8", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8726757, "90.00 percentile latency (ns)": 10012693, "90th percentile latency (ns)": 10012693, "95.00 percentile latency (ns)": 12783850, "97.00 percentile latency (ns)": 24543995, "99.00 percentile latency (ns)": 92362757, "99.90 percentile latency (ns)": 175647980, "Max latency (ns)": 198068790, "Mean latency (ns)": 10834358, "Min duration satisfied": "Yes", "Min latency (ns)": 7548407, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 91.98, "QPS w/o loadgen overhead": 92.3, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.226, "characteristics.good": 29613, "characteristics.samples_per_second": 92.3, "characteristics.samples_per_second.normalized_per_core": 0.7691666666666667, "characteristics.samples_per_second.normalized_per_processor": 92.3, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "8c0116654b17a5f0", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 16724480, "90.00 percentile latency (ns)": 21916435, "90th percentile latency (ns)": 21916435, "95.00 percentile latency (ns)": 40067168, "97.00 percentile latency (ns)": 64680357, "99.00 percentile latency (ns)": 106111078, "99.90 percentile latency (ns)": 120064113, "Max latency (ns)": 129921283, "Mean latency (ns)": 20477601, "Min duration satisfied": "Yes", "Min latency (ns)": 15601850, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 48.73, "QPS w/o loadgen overhead": 48.83, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.248, "characteristics.good": 34124, "characteristics.samples_per_second": 48.83, "characteristics.samples_per_second.normalized_per_core": 0.40691666666666665, "characteristics.samples_per_second.normalized_per_processor": 48.83, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 47.619, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "630cb347ee5647eb", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7609864, "90.00 percentile latency (ns)": 8780318, "90th percentile latency (ns)": 8780318, "95.00 percentile latency (ns)": 10817319, "97.00 percentile latency (ns)": 20330877, "99.00 percentile latency (ns)": 59454596, "99.90 percentile latency (ns)": 173430501, "Max latency (ns)": 203213816, "Mean latency (ns)": 9482643, "Min duration satisfied": "Yes", "Min latency (ns)": 6854443, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 105.02, "QPS w/o loadgen overhead": 105.46, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.12, "characteristics.good": 29560, "characteristics.samples_per_second": 105.46, "characteristics.samples_per_second.normalized_per_core": 0.8788333333333332, "characteristics.samples_per_second.normalized_per_processor": 105.46, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "14bf9deaa19fd5c9", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7953172, "90.00 percentile latency (ns)": 9543919, "90th percentile latency (ns)": 9543919, "95.00 percentile latency (ns)": 12537004, "97.00 percentile latency (ns)": 25609537, "99.00 percentile latency (ns)": 79335043, "99.90 percentile latency (ns)": 105626994, "Max latency (ns)": 127819786, "Mean latency (ns)": 9946099, "Min duration satisfied": "Yes", "Min latency (ns)": 7124544, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 100.13, "QPS w/o loadgen overhead": 100.54, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.46, "characteristics.good": 29230, "characteristics.samples_per_second": 100.54, "characteristics.samples_per_second.normalized_per_core": 0.8378333333333334, "characteristics.samples_per_second.normalized_per_processor": 100.54, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "20cd7007847e4069", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 28878146, "90.00 percentile latency (ns)": 43218781, "90th percentile latency (ns)": 43218781, "95.00 percentile latency (ns)": 56789712, "97.00 percentile latency (ns)": 83915530, "99.00 percentile latency (ns)": 174956463, "99.90 percentile latency (ns)": 202318243, "Max latency (ns)": 160739966122, "Mean latency (ns)": 46022134, "Min duration satisfied": "Yes", "Min latency (ns)": 27275443, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 21.72, "QPS w/o loadgen overhead": 21.73, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.91, "characteristics.good": 36455, "characteristics.samples_per_second": 21.73, "characteristics.samples_per_second.normalized_per_core": 0.18108333333333335, "characteristics.samples_per_second.normalized_per_processor": 21.73, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 27.027, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "96f184a928846389", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 26462839, "90.00 percentile latency (ns)": 49678656, "90th percentile latency (ns)": 49678656, "95.00 percentile latency (ns)": 101690992, "97.00 percentile latency (ns)": 152968485, "99.00 percentile latency (ns)": 191794394, "99.90 percentile latency (ns)": 211942288, "Max latency (ns)": 228234846, "Mean latency (ns)": 36858679, "Min duration satisfied": "Yes", "Min latency (ns)": 25704804, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 27.09, "QPS w/o loadgen overhead": 27.13, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.542, "characteristics.good": 35271, "characteristics.samples_per_second": 27.13, "characteristics.samples_per_second.normalized_per_core": 0.22608333333333333, "characteristics.samples_per_second.normalized_per_processor": 27.13, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 25.641, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "3ec6f6a4418686d4", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9191907, "90.00 percentile latency (ns)": 10390383, "90th percentile latency (ns)": 10390383, "95.00 percentile latency (ns)": 13079545, "97.00 percentile latency (ns)": 25384041, "99.00 percentile latency (ns)": 84731691, "99.90 percentile latency (ns)": 175553391, "Max latency (ns)": 204941029, "Mean latency (ns)": 11095756, "Min duration satisfied": "Yes", "Min latency (ns)": 7801815, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 89.86, "QPS w/o loadgen overhead": 90.12, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.594, "characteristics.good": 28297, "characteristics.samples_per_second": 90.12, "characteristics.samples_per_second.normalized_per_core": 0.751, "characteristics.samples_per_second.normalized_per_processor": 90.12, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "83def7ed6ab430ab", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 16361052, "90.00 percentile latency (ns)": 21001183, "90th percentile latency (ns)": 21001183, "95.00 percentile latency (ns)": 38405818, "97.00 percentile latency (ns)": 62696870, "99.00 percentile latency (ns)": 155922213, "99.90 percentile latency (ns)": 196033192, "Max latency (ns)": 214988713, "Mean latency (ns)": 20832189, "Min duration satisfied": "Yes", "Min latency (ns)": 14747262, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 47.91, "QPS w/o loadgen overhead": 48.0, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.82, "characteristics.good": 33910, "characteristics.samples_per_second": 48.0, "characteristics.samples_per_second.normalized_per_core": 0.4, "characteristics.samples_per_second.normalized_per_processor": 48.0, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "34fd5a911b4004ec", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4947828, "90.00 percentile latency (ns)": 30516176, "90th percentile latency (ns)": 30516176, "95.00 percentile latency (ns)": 64529432, "97.00 percentile latency (ns)": 76901918, "99.00 percentile latency (ns)": 87061617, "99.90 percentile latency (ns)": 99310563, "Max latency (ns)": 117784155, "Mean latency (ns)": 12001121, "Min duration satisfied": "Yes", "Min latency (ns)": 4612120, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 82.99, "QPS w/o loadgen overhead": 83.33, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 44.116, "characteristics.good": 22058, "characteristics.samples_per_second": 83.33, "characteristics.samples_per_second.normalized_per_core": 0.6944166666666667, "characteristics.samples_per_second.normalized_per_processor": 83.33, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "53e4293a6c868a77", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4009547, "90.00 percentile latency (ns)": 4341171, "90th percentile latency (ns)": 4341171, "95.00 percentile latency (ns)": 5397591, "97.00 percentile latency (ns)": 6144258, "99.00 percentile latency (ns)": 30616861, "99.90 percentile latency (ns)": 87587599, "Max latency (ns)": 106809644, "Mean latency (ns)": 4692802, "Min duration satisfied": "Yes", "Min latency (ns)": 3190546, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 211.72, "QPS w/o loadgen overhead": 213.09, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 45.494, "characteristics.good": 22747, "characteristics.samples_per_second": 213.09, "characteristics.samples_per_second.normalized_per_core": 1.77575, "characteristics.samples_per_second.normalized_per_processor": 213.09, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d674d5eb2218c607", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 49632304, "90.00 percentile latency (ns)": 118952929, "90th percentile latency (ns)": 118952929, "95.00 percentile latency (ns)": 135847991, "97.00 percentile latency (ns)": 140272624, "99.00 percentile latency (ns)": 147364494, "99.90 percentile latency (ns)": 156995272, "Max latency (ns)": 183180558, "Mean latency (ns)": 62881535, "Min duration satisfied": "Yes", "Min latency (ns)": 44683289, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 15.89, "QPS w/o loadgen overhead": 15.9, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.626, "characteristics.good": 35313, "characteristics.samples_per_second": 15.9, "characteristics.samples_per_second.normalized_per_core": 0.1325, "characteristics.samples_per_second.normalized_per_processor": 15.9, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.625, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "385c33bad9bedcd6", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11156545, "90.00 percentile latency (ns)": 13505046, "90th percentile latency (ns)": 13505046, "95.00 percentile latency (ns)": 23485296, "97.00 percentile latency (ns)": 34850675, "99.00 percentile latency (ns)": 116066748, "99.90 percentile latency (ns)": 181382611, "Max latency (ns)": 208874787, "Mean latency (ns)": 14082801, "Min duration satisfied": "Yes", "Min latency (ns)": 10117629, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 70.81, "QPS w/o loadgen overhead": 71.01, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.454, "characteristics.good": 31727, "characteristics.samples_per_second": 71.01, "characteristics.samples_per_second.normalized_per_core": 0.59175, "characteristics.samples_per_second.normalized_per_processor": 71.01, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "f3030c7b46b1b026", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 22752264, "90.00 percentile latency (ns)": 33643121, "90th percentile latency (ns)": 33643121, "95.00 percentile latency (ns)": 53766398, "97.00 percentile latency (ns)": 85254137, "99.00 percentile latency (ns)": 114690295, "99.90 percentile latency (ns)": 126756253, "Max latency (ns)": 136000570, "Mean latency (ns)": 27371532, "Min duration satisfied": "Yes", "Min latency (ns)": 21312368, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 36.48, "QPS w/o loadgen overhead": 36.53, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.04, "characteristics.good": 35020, "characteristics.samples_per_second": 36.53, "characteristics.samples_per_second.normalized_per_core": 0.30441666666666667, "characteristics.samples_per_second.normalized_per_processor": 36.53, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 37.037, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "8b2bb37218caf811", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 90416167, "90.00 percentile latency (ns)": 231711674, "90th percentile latency (ns)": 231711674, "95.00 percentile latency (ns)": 250833741, "97.00 percentile latency (ns)": 259360061, "99.00 percentile latency (ns)": 271404709, "99.90 percentile latency (ns)": 291127981, "Max latency (ns)": 300736761, "Mean latency (ns)": 126881152, "Min duration satisfied": "Yes", "Min latency (ns)": 71903148, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.88, "QPS w/o loadgen overhead": 7.88, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 74.124, "characteristics.good": 37062, "characteristics.samples_per_second": 7.88, "characteristics.samples_per_second.normalized_per_core": 0.06566666666666666, "characteristics.samples_per_second.normalized_per_processor": 7.88, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite2-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 9.25926, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "a3d425dbb09d0554", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6970839, "90.00 percentile latency (ns)": 8088215, "90th percentile latency (ns)": 8088215, "95.00 percentile latency (ns)": 9513299, "97.00 percentile latency (ns)": 20565140, "99.00 percentile latency (ns)": 74125394, "99.90 percentile latency (ns)": 102002326, "Max latency (ns)": 119584002, "Mean latency (ns)": 8611250, "Min duration satisfied": "Yes", "Min latency (ns)": 6090588, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 115.67, "QPS w/o loadgen overhead": 116.13, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.444, "characteristics.good": 29722, "characteristics.samples_per_second": 116.13, "characteristics.samples_per_second.normalized_per_core": 0.96775, "characteristics.samples_per_second.normalized_per_processor": 116.13, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d55d46be98b93580", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8955641, "90.00 percentile latency (ns)": 10188527, "90th percentile latency (ns)": 10188527, "95.00 percentile latency (ns)": 14617782, "97.00 percentile latency (ns)": 29265301, "99.00 percentile latency (ns)": 86032251, "99.90 percentile latency (ns)": 107130499, "Max latency (ns)": 126045026, "Mean latency (ns)": 10929574, "Min duration satisfied": "Yes", "Min latency (ns)": 7845267, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 91.19, "QPS w/o loadgen overhead": 91.49, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.456, "characteristics.good": 30728, "characteristics.samples_per_second": 91.49, "characteristics.samples_per_second.normalized_per_core": 0.7624166666666666, "characteristics.samples_per_second.normalized_per_processor": 91.49, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "dff64e7fbff4106b", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10471997, "90.00 percentile latency (ns)": 14271455, "90th percentile latency (ns)": 14271455, "95.00 percentile latency (ns)": 26987981, "97.00 percentile latency (ns)": 47446276, "99.00 percentile latency (ns)": 151249767, "99.90 percentile latency (ns)": 185743703, "Max latency (ns)": 213258051, "Mean latency (ns)": 14820536, "Min duration satisfied": "Yes", "Min latency (ns)": 9934163, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 67.22, "QPS w/o loadgen overhead": 67.47, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 58.528, "characteristics.good": 29264, "characteristics.samples_per_second": 67.47, "characteristics.samples_per_second.normalized_per_core": 0.56225, "characteristics.samples_per_second.normalized_per_processor": 67.47, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 62.5, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "747bb3f822c43bea", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 28615043, "90.00 percentile latency (ns)": 179620350, "90th percentile latency (ns)": 179620350, "95.00 percentile latency (ns)": 189009398, "97.00 percentile latency (ns)": 193102292, "99.00 percentile latency (ns)": 200601050, "99.90 percentile latency (ns)": 213657843, "Max latency (ns)": 221220459, "Mean latency (ns)": 65094858, "Min duration satisfied": "Yes", "Min latency (ns)": 19996934, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 15.35, "QPS w/o loadgen overhead": 15.36, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.036, "characteristics.good": 33018, "characteristics.samples_per_second": 15.36, "characteristics.samples_per_second.normalized_per_core": 0.128, "characteristics.samples_per_second.normalized_per_processor": 15.36, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.1515, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "dcfa09d8421085c2", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 106589932, "90.00 percentile latency (ns)": 203312804, "90th percentile latency (ns)": 203312804, "95.00 percentile latency (ns)": 251745383, "97.00 percentile latency (ns)": 268920041, "99.00 percentile latency (ns)": 284618617, "99.90 percentile latency (ns)": 305475888, "Max latency (ns)": 324681961, "Mean latency (ns)": 124006273, "Min duration satisfied": "Yes", "Min latency (ns)": 92041550, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 8.06, "QPS w/o loadgen overhead": 8.06, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.642, "characteristics.good": 38321, "characteristics.samples_per_second": 8.06, "characteristics.samples_per_second.normalized_per_core": 0.06716666666666667, "characteristics.samples_per_second.normalized_per_processor": 8.06, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite1-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 7.69231, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "f483053ed32b20a5", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 21818130, "90.00 percentile latency (ns)": 33610496, "90th percentile latency (ns)": 33610496, "95.00 percentile latency (ns)": 58053034, "97.00 percentile latency (ns)": 108601529, "99.00 percentile latency (ns)": 178887184, "99.90 percentile latency (ns)": 203555712, "Max latency (ns)": 231779097, "Mean latency (ns)": 27950052, "Min duration satisfied": "Yes", "Min latency (ns)": 19566752, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 35.73, "QPS w/o loadgen overhead": 35.78, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.604, "characteristics.good": 34802, "characteristics.samples_per_second": 35.78, "characteristics.samples_per_second.normalized_per_core": 0.2981666666666667, "characteristics.samples_per_second.normalized_per_processor": 35.78, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 35.7143, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4e0036b64a529da4", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 45189077, "90.00 percentile latency (ns)": 122187904, "90th percentile latency (ns)": 122187904, "95.00 percentile latency (ns)": 130905815, "97.00 percentile latency (ns)": 134444035, "99.00 percentile latency (ns)": 140527160, "99.90 percentile latency (ns)": 150289542, "Max latency (ns)": 160310028, "Mean latency (ns)": 60152709, "Min duration satisfied": "Yes", "Min latency (ns)": 39931789, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.61, "QPS w/o loadgen overhead": 16.62, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.262, "characteristics.good": 34631, "characteristics.samples_per_second": 16.62, "characteristics.samples_per_second.normalized_per_core": 0.1385, "characteristics.samples_per_second.normalized_per_processor": 16.62, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 16.129, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "bd142c7346228398", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 21614571, "90.00 percentile latency (ns)": 40648506, "90th percentile latency (ns)": 40648506, "95.00 percentile latency (ns)": 84663372, "97.00 percentile latency (ns)": 137631588, "99.00 percentile latency (ns)": 184040921, "99.90 percentile latency (ns)": 202353444, "Max latency (ns)": 225901545, "Mean latency (ns)": 30785840, "Min duration satisfied": "Yes", "Min latency (ns)": 20811895, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 32.43, "QPS w/o loadgen overhead": 32.48, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.734, "characteristics.good": 34367, "characteristics.samples_per_second": 32.48, "characteristics.samples_per_second.normalized_per_core": 0.27066666666666667, "characteristics.samples_per_second.normalized_per_processor": 32.48, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 27.7778, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "90a29f17d2650a26", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 43281321, "90.00 percentile latency (ns)": 127912213, "90th percentile latency (ns)": 127912213, "95.00 percentile latency (ns)": 131969344, "97.00 percentile latency (ns)": 134249616, "99.00 percentile latency (ns)": 139825670, "99.90 percentile latency (ns)": 150726574, "Max latency (ns)": 157960418, "Mean latency (ns)": 62163370, "Min duration satisfied": "Yes", "Min latency (ns)": 15469162, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.08, "QPS w/o loadgen overhead": 16.09, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.33, "characteristics.good": 31665, "characteristics.samples_per_second": 16.09, "characteristics.samples_per_second.normalized_per_core": 0.13408333333333333, "characteristics.samples_per_second.normalized_per_processor": 16.09, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.1515, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "39ca1fa33de23200", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10282142, "90.00 percentile latency (ns)": 90233335, "90th percentile latency (ns)": 90233335, "95.00 percentile latency (ns)": 97159009, "97.00 percentile latency (ns)": 99925154, "99.00 percentile latency (ns)": 104573156, "99.90 percentile latency (ns)": 111926093, "Max latency (ns)": 122452358, "Mean latency (ns)": 27068846, "Min duration satisfied": "Yes", "Min latency (ns)": 8785888, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 36.84, "QPS w/o loadgen overhead": 36.94, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 57.918, "characteristics.good": 28959, "characteristics.samples_per_second": 36.94, "characteristics.samples_per_second.normalized_per_core": 0.3078333333333333, "characteristics.samples_per_second.normalized_per_processor": 36.94, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "1a1b4a70f42c8aca", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 67743731, "90.00 percentile latency (ns)": 122530897, "90th percentile latency (ns)": 122530897, "95.00 percentile latency (ns)": 143257613, "97.00 percentile latency (ns)": 148491169, "99.00 percentile latency (ns)": 156266209, "99.90 percentile latency (ns)": 168553925, "Max latency (ns)": 176712140, "Mean latency (ns)": 76491142, "Min duration satisfied": "Yes", "Min latency (ns)": 58807686, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 13.07, "QPS w/o loadgen overhead": 13.07, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.676, "characteristics.good": 35838, "characteristics.samples_per_second": 13.07, "characteristics.samples_per_second.normalized_per_core": 0.10891666666666668, "characteristics.samples_per_second.normalized_per_processor": 13.07, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.987, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "cce8e3e934c44658", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3745872, "90.00 percentile latency (ns)": 4090914, "90th percentile latency (ns)": 4090914, "95.00 percentile latency (ns)": 4971455, "97.00 percentile latency (ns)": 5926953, "99.00 percentile latency (ns)": 23751563, "99.90 percentile latency (ns)": 120678713, "Max latency (ns)": 178539681, "Mean latency (ns)": 4456605, "Min duration satisfied": "Yes", "Min latency (ns)": 3191414, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 222.63, "QPS w/o loadgen overhead": 224.39, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 43.468, "characteristics.good": 21734, "characteristics.samples_per_second": 224.39, "characteristics.samples_per_second.normalized_per_core": 1.8699166666666664, "characteristics.samples_per_second.normalized_per_processor": 224.39, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "45ade85888cc400b", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 503220677, "90.00 percentile latency (ns)": 554081097, "90th percentile latency (ns)": 554081097, "95.00 percentile latency (ns)": 564939845, "97.00 percentile latency (ns)": 570244085, "99.00 percentile latency (ns)": 581902871, "99.90 percentile latency (ns)": 593908142, "Max latency (ns)": 600536536, "Mean latency (ns)": 497972933, "Min duration satisfied": "Yes", "Min latency (ns)": 318925265, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.01, "QPS w/o loadgen overhead": 2.01, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 80.676, "characteristics.good": 40338, "characteristics.samples_per_second": 2.01, "characteristics.samples_per_second.normalized_per_core": 0.016749999999999998, "characteristics.samples_per_second.normalized_per_processor": 2.01, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite4-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.04082, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "18e2ca4a16922acc", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20270503, "90.00 percentile latency (ns)": 42403287, "90th percentile latency (ns)": 42403287, "95.00 percentile latency (ns)": 107251446, "97.00 percentile latency (ns)": 151866323, "99.00 percentile latency (ns)": 177106781, "99.90 percentile latency (ns)": 190038715, "Max latency (ns)": 202797795, "Mean latency (ns)": 30389721, "Min duration satisfied": "Yes", "Min latency (ns)": 19264839, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 32.85, "QPS w/o loadgen overhead": 32.91, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.43, "characteristics.good": 35215, "characteristics.samples_per_second": 32.91, "characteristics.samples_per_second.normalized_per_core": 0.27425, "characteristics.samples_per_second.normalized_per_processor": 32.91, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 31.25, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "7b2cd04eb19c313e", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 17485440, "90.00 percentile latency (ns)": 23685421, "90th percentile latency (ns)": 23685421, "95.00 percentile latency (ns)": 42668252, "97.00 percentile latency (ns)": 74788779, "99.00 percentile latency (ns)": 166965010, "99.90 percentile latency (ns)": 198888353, "Max latency (ns)": 226422449, "Mean latency (ns)": 22438926, "Min duration satisfied": "Yes", "Min latency (ns)": 15654939, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 44.48, "QPS w/o loadgen overhead": 44.57, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.474, "characteristics.good": 33737, "characteristics.samples_per_second": 44.57, "characteristics.samples_per_second.normalized_per_core": 0.3714166666666667, "characteristics.samples_per_second.normalized_per_processor": 44.57, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "0ad9df4c2e9d137e", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 14686292, "90.00 percentile latency (ns)": 19512204, "90th percentile latency (ns)": 19512204, "95.00 percentile latency (ns)": 39265919, "97.00 percentile latency (ns)": 72863289, "99.00 percentile latency (ns)": 106072431, "99.90 percentile latency (ns)": 120435576, "Max latency (ns)": 136243646, "Mean latency (ns)": 18624130, "Min duration satisfied": "Yes", "Min latency (ns)": 13508544, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 53.57, "QPS w/o loadgen overhead": 53.69, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.24, "characteristics.good": 33620, "characteristics.samples_per_second": 53.69, "characteristics.samples_per_second.normalized_per_core": 0.44741666666666663, "characteristics.samples_per_second.normalized_per_processor": 53.69, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 55.5556, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "94ec2b320095c5e6", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 35300165, "90.00 percentile latency (ns)": 115702354, "90th percentile latency (ns)": 115702354, "95.00 percentile latency (ns)": 119589031, "97.00 percentile latency (ns)": 122172321, "99.00 percentile latency (ns)": 126745348, "99.90 percentile latency (ns)": 134645929, "Max latency (ns)": 141075289, "Mean latency (ns)": 56934882, "Min duration satisfied": "Yes", "Min latency (ns)": 23172888, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.54, "QPS w/o loadgen overhead": 17.56, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.348, "characteristics.good": 32174, "characteristics.samples_per_second": 17.56, "characteristics.samples_per_second.normalized_per_core": 0.14633333333333332, "characteristics.samples_per_second.normalized_per_processor": 17.56, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.6582, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "decdb3cfaf5481d8", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 36138644, "90.00 percentile latency (ns)": 110737599, "90th percentile latency (ns)": 110737599, "95.00 percentile latency (ns)": 114342512, "97.00 percentile latency (ns)": 116566262, "99.00 percentile latency (ns)": 121046175, "99.90 percentile latency (ns)": 126537171, "Max latency (ns)": 132045668, "Mean latency (ns)": 55469790, "Min duration satisfied": "Yes", "Min latency (ns)": 17048489, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 18.0, "QPS w/o loadgen overhead": 18.03, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.246, "characteristics.good": 31623, "characteristics.samples_per_second": 18.03, "characteristics.samples_per_second.normalized_per_core": 0.15025000000000002, "characteristics.samples_per_second.normalized_per_processor": 18.03, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 16.9492, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "2204245d66784a3f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20549698, "90.00 percentile latency (ns)": 169187225, "90th percentile latency (ns)": 169187225, "95.00 percentile latency (ns)": 179325087, "97.00 percentile latency (ns)": 184369585, "99.00 percentile latency (ns)": 191780568, "99.90 percentile latency (ns)": 205226531, "Max latency (ns)": 220824785, "Mean latency (ns)": 57280459, "Min duration satisfied": "Yes", "Min latency (ns)": 14240071, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.44, "QPS w/o loadgen overhead": 17.46, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.668, "characteristics.good": 30834, "characteristics.samples_per_second": 17.46, "characteristics.samples_per_second.normalized_per_core": 0.14550000000000002, "characteristics.samples_per_second.normalized_per_processor": 17.46, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.9254, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "9a36d84f3a724462", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 27665774, "90.00 percentile latency (ns)": 113300778, "90th percentile latency (ns)": 113300778, "95.00 percentile latency (ns)": 117596512, "97.00 percentile latency (ns)": 120170384, "99.00 percentile latency (ns)": 124628798, "99.90 percentile latency (ns)": 132271001, "Max latency (ns)": 139730455, "Mean latency (ns)": 49407957, "Min duration satisfied": "Yes", "Min latency (ns)": 22991778, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.21, "QPS w/o loadgen overhead": 20.24, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.384, "characteristics.good": 33192, "characteristics.samples_per_second": 20.24, "characteristics.samples_per_second.normalized_per_core": 0.16866666666666666, "characteristics.samples_per_second.normalized_per_processor": 20.24, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 21.2766, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "324202b74b0fbf72", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4635157, "90.00 percentile latency (ns)": 5126905, "90th percentile latency (ns)": 5126905, "95.00 percentile latency (ns)": 6362987, "97.00 percentile latency (ns)": 7819274, "99.00 percentile latency (ns)": 32775392, "99.90 percentile latency (ns)": 148147406, "Max latency (ns)": 181068610, "Mean latency (ns)": 5721086, "Min duration satisfied": "Yes", "Min latency (ns)": 4075741, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 173.72, "QPS w/o loadgen overhead": 174.79, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.11, "characteristics.good": 24555, "characteristics.samples_per_second": 174.79, "characteristics.samples_per_second.normalized_per_core": 1.4565833333333333, "characteristics.samples_per_second.normalized_per_processor": 174.79, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d048f0af624d1a39", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9379979, "90.00 percentile latency (ns)": 10810885, "90th percentile latency (ns)": 10810885, "95.00 percentile latency (ns)": 16302379, "97.00 percentile latency (ns)": 29543437, "99.00 percentile latency (ns)": 105018102, "99.90 percentile latency (ns)": 179386278, "Max latency (ns)": 213687008, "Mean latency (ns)": 11901342, "Min duration satisfied": "Yes", "Min latency (ns)": 8385109, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 83.74, "QPS w/o loadgen overhead": 84.02, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 62.07, "characteristics.good": 31035, "characteristics.samples_per_second": 84.02, "characteristics.samples_per_second.normalized_per_core": 0.7001666666666666, "characteristics.samples_per_second.normalized_per_processor": 84.02, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "1fca7841a5b90a6b", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 22627786, "90.00 percentile latency (ns)": 168853453, "90th percentile latency (ns)": 168853453, "95.00 percentile latency (ns)": 178868239, "97.00 percentile latency (ns)": 182668829, "99.00 percentile latency (ns)": 188494870, "99.90 percentile latency (ns)": 197519533, "Max latency (ns)": 203509281, "Mean latency (ns)": 56647477, "Min duration satisfied": "Yes", "Min latency (ns)": 14391705, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.64, "QPS w/o loadgen overhead": 17.65, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.224, "characteristics.good": 30612, "characteristics.samples_per_second": 17.65, "characteristics.samples_per_second.normalized_per_core": 0.14708333333333332, "characteristics.samples_per_second.normalized_per_processor": 17.65, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.0845, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "04d46909761c4bfa", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2749544, "90.00 percentile latency (ns)": 3015543, "90th percentile latency (ns)": 3015543, "95.00 percentile latency (ns)": 3563294, "97.00 percentile latency (ns)": 4429836, "99.00 percentile latency (ns)": 16291178, "99.90 percentile latency (ns)": 83361023, "Max latency (ns)": 106095271, "Mean latency (ns)": 3233656, "Min duration satisfied": "Yes", "Min latency (ns)": 2188960, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 306.51, "QPS w/o loadgen overhead": 309.25, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 42.384, "characteristics.good": 21192, "characteristics.samples_per_second": 309.25, "characteristics.samples_per_second.normalized_per_core": 2.5770833333333334, "characteristics.samples_per_second.normalized_per_processor": 309.25, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "0159638ed4187b68", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 96113123, "90.00 percentile latency (ns)": 228931011, "90th percentile latency (ns)": 228931011, "95.00 percentile latency (ns)": 239924777, "97.00 percentile latency (ns)": 245077423, "99.00 percentile latency (ns)": 253878340, "99.90 percentile latency (ns)": 271382449, "Max latency (ns)": 284135526, "Mean latency (ns)": 125464985, "Min duration satisfied": "Yes", "Min latency (ns)": 75414155, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.97, "QPS w/o loadgen overhead": 7.97, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.012, "characteristics.good": 37506, "characteristics.samples_per_second": 7.97, "characteristics.samples_per_second.normalized_per_core": 0.06641666666666667, "characteristics.samples_per_second.normalized_per_processor": 7.97, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.3-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.3-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.3_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 7.46269, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "84e51a8a51452bc9", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 12036802, "90.00 percentile latency (ns)": 21483019, "90th percentile latency (ns)": 21483019, "95.00 percentile latency (ns)": 44875567, "97.00 percentile latency (ns)": 92256143, "99.00 percentile latency (ns)": 122513130, "99.90 percentile latency (ns)": 136257003, "Max latency (ns)": 144366340, "Mean latency (ns)": 17455047, "Min duration satisfied": "Yes", "Min latency (ns)": 11346125, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 57.24, "QPS w/o loadgen overhead": 57.29, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.486, "characteristics.good": 32243, "characteristics.samples_per_second": 57.29, "characteristics.samples_per_second.normalized_per_core": 0.47741666666666666, "characteristics.samples_per_second.normalized_per_processor": 57.29, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 41.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d79f316b7bdd36a5", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 243848970, "90.00 percentile latency (ns)": 298329335, "90th percentile latency (ns)": 298329335, "95.00 percentile latency (ns)": 307291037, "97.00 percentile latency (ns)": 312102970, "99.00 percentile latency (ns)": 321092681, "99.90 percentile latency (ns)": 332047095, "Max latency (ns)": 338177891, "Mean latency (ns)": 231050807, "Min duration satisfied": "Yes", "Min latency (ns)": 122919508, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 4.33, "QPS w/o loadgen overhead": 4.33, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 77.542, "characteristics.good": 38771, "characteristics.samples_per_second": 4.33, "characteristics.samples_per_second.normalized_per_core": 0.036083333333333335, "characteristics.samples_per_second.normalized_per_processor": 4.33, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite2-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 4.1841, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "2240e841e0b409b7", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4827073, "90.00 percentile latency (ns)": 5459405, "90th percentile latency (ns)": 5459405, "95.00 percentile latency (ns)": 6798152, "97.00 percentile latency (ns)": 8920026, "99.00 percentile latency (ns)": 53805415, "99.90 percentile latency (ns)": 97786919, "Max latency (ns)": 117296512, "Mean latency (ns)": 6009059, "Min duration satisfied": "Yes", "Min latency (ns)": 4177826, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 165.42, "QPS w/o loadgen overhead": 166.42, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.75, "characteristics.good": 28375, "characteristics.samples_per_second": 166.42, "characteristics.samples_per_second.normalized_per_core": 1.3868333333333331, "characteristics.samples_per_second.normalized_per_processor": 166.42, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "8ab37c7f6d15948a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6483459, "90.00 percentile latency (ns)": 7583042, "90th percentile latency (ns)": 7583042, "95.00 percentile latency (ns)": 9293376, "97.00 percentile latency (ns)": 17840335, "99.00 percentile latency (ns)": 63090131, "99.90 percentile latency (ns)": 165986641, "Max latency (ns)": 192259104, "Mean latency (ns)": 8328760, "Min duration satisfied": "Yes", "Min latency (ns)": 5775876, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 119.54, "QPS w/o loadgen overhead": 120.07, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 57.002, "characteristics.good": 28501, "characteristics.samples_per_second": 120.07, "characteristics.samples_per_second.normalized_per_core": 1.0005833333333334, "characteristics.samples_per_second.normalized_per_processor": 120.07, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "9b88c81265e3a990", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 99125647, "90.00 percentile latency (ns)": 232380799, "90th percentile latency (ns)": 232380799, "95.00 percentile latency (ns)": 244867161, "97.00 percentile latency (ns)": 249835240, "99.00 percentile latency (ns)": 259635698, "99.90 percentile latency (ns)": 279420711, "Max latency (ns)": 289281524, "Mean latency (ns)": 127163377, "Min duration satisfied": "Yes", "Min latency (ns)": 80729326, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.86, "QPS w/o loadgen overhead": 7.86, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.69, "characteristics.good": 37845, "characteristics.samples_per_second": 7.86, "characteristics.samples_per_second.normalized_per_core": 0.0655, "characteristics.samples_per_second.normalized_per_processor": 7.86, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.4-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.4-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.4_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 7.8125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4c4a8151f7643483", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11721238, "90.00 percentile latency (ns)": 111787400, "90th percentile latency (ns)": 111787400, "95.00 percentile latency (ns)": 147639398, "97.00 percentile latency (ns)": 157290146, "99.00 percentile latency (ns)": 167054418, "99.90 percentile latency (ns)": 182406889, "Max latency (ns)": 192920314, "Mean latency (ns)": 37716950, "Min duration satisfied": "Yes", "Min latency (ns)": 7497604, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 26.48, "QPS w/o loadgen overhead": 26.51, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 52.446, "characteristics.good": 26223, "characteristics.samples_per_second": 26.51, "characteristics.samples_per_second.normalized_per_core": 0.22091666666666668, "characteristics.samples_per_second.normalized_per_processor": 26.51, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.2558, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "74515bcb1a89fc34", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 43988211, "90.00 percentile latency (ns)": 178022175, "90th percentile latency (ns)": 178022175, "95.00 percentile latency (ns)": 188232770, "97.00 percentile latency (ns)": 192904211, "99.00 percentile latency (ns)": 199172376, "99.90 percentile latency (ns)": 210536332, "Max latency (ns)": 230974525, "Mean latency (ns)": 72854307, "Min duration satisfied": "Yes", "Min latency (ns)": 36749680, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 13.72, "QPS w/o loadgen overhead": 13.73, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.724, "characteristics.good": 36862, "characteristics.samples_per_second": 13.73, "characteristics.samples_per_second.normalized_per_core": 0.11441666666666667, "characteristics.samples_per_second.normalized_per_processor": 13.73, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.0482, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "75f4010b388b2236", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 33457731, "90.00 percentile latency (ns)": 115689280, "90th percentile latency (ns)": 115689280, "95.00 percentile latency (ns)": 120235867, "97.00 percentile latency (ns)": 122501329, "99.00 percentile latency (ns)": 126515372, "99.90 percentile latency (ns)": 133160611, "Max latency (ns)": 150520983, "Mean latency (ns)": 55294256, "Min duration satisfied": "Yes", "Min latency (ns)": 24161973, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 18.06, "QPS w/o loadgen overhead": 18.09, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.604, "characteristics.good": 33302, "characteristics.samples_per_second": 18.09, "characteristics.samples_per_second.normalized_per_core": 0.15075, "characteristics.samples_per_second.normalized_per_processor": 18.09, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.873, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "31814fd19dec8076", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 38755458, "90.00 percentile latency (ns)": 121937577, "90th percentile latency (ns)": 121937577, "95.00 percentile latency (ns)": 128653195, "97.00 percentile latency (ns)": 132230979, "99.00 percentile latency (ns)": 137968642, "99.90 percentile latency (ns)": 146885468, "Max latency (ns)": 152673921, "Mean latency (ns)": 58018478, "Min duration satisfied": "Yes", "Min latency (ns)": 34360921, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.22, "QPS w/o loadgen overhead": 17.24, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.106, "characteristics.good": 34553, "characteristics.samples_per_second": 17.24, "characteristics.samples_per_second.normalized_per_core": 0.14366666666666666, "characteristics.samples_per_second.normalized_per_processor": 17.24, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 16.9492, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "d8f544379ebfe4fd", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 19350035, "90.00 percentile latency (ns)": 160373772, "90th percentile latency (ns)": 160373772, "95.00 percentile latency (ns)": 172283924, "97.00 percentile latency (ns)": 177124693, "99.00 percentile latency (ns)": 185596817, "99.90 percentile latency (ns)": 197655133, "Max latency (ns)": 206874427, "Mean latency (ns)": 51139182, "Min duration satisfied": "Yes", "Min latency (ns)": 11498079, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 19.53, "QPS w/o loadgen overhead": 19.55, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.822, "characteristics.good": 29911, "characteristics.samples_per_second": 19.55, "characteristics.samples_per_second.normalized_per_core": 0.16291666666666668, "characteristics.samples_per_second.normalized_per_processor": 19.55, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.7059, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "f9bf96847bbb010c", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20383798, "90.00 percentile latency (ns)": 123877189, "90th percentile latency (ns)": 123877189, "95.00 percentile latency (ns)": 133162654, "97.00 percentile latency (ns)": 136264384, "99.00 percentile latency (ns)": 141942488, "99.90 percentile latency (ns)": 150222066, "Max latency (ns)": 173610610, "Mean latency (ns)": 40443163, "Min duration satisfied": "Yes", "Min latency (ns)": 19249502, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 24.72, "QPS w/o loadgen overhead": 24.73, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.336, "characteristics.good": 34168, "characteristics.samples_per_second": 24.73, "characteristics.samples_per_second.normalized_per_core": 0.20608333333333334, "characteristics.samples_per_second.normalized_per_processor": 24.73, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 19.6078, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "a8ac5baa4fd411b8", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 32196498, "90.00 percentile latency (ns)": 160903694, "90th percentile latency (ns)": 160903694, "95.00 percentile latency (ns)": 188515520, "97.00 percentile latency (ns)": 194370447, "99.00 percentile latency (ns)": 202270529, "99.90 percentile latency (ns)": 216298568, "Max latency (ns)": 237935194, "Mean latency (ns)": 58375633, "Min duration satisfied": "Yes", "Min latency (ns)": 29733663, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.12, "QPS w/o loadgen overhead": 17.13, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.834, "characteristics.good": 34417, "characteristics.samples_per_second": 17.13, "characteristics.samples_per_second.normalized_per_core": 0.14275, "characteristics.samples_per_second.normalized_per_processor": 17.13, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 13.8889, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "3465dad2a4702dd8", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 37749101, "90.00 percentile latency (ns)": 87996361, "90th percentile latency (ns)": 87996361, "95.00 percentile latency (ns)": 151407989, "97.00 percentile latency (ns)": 175164273, "99.00 percentile latency (ns)": 192469186, "99.90 percentile latency (ns)": 202696471, "Max latency (ns)": 210160995, "Mean latency (ns)": 51705064, "Min duration satisfied": "Yes", "Min latency (ns)": 34763554, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 19.32, "QPS w/o loadgen overhead": 19.34, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.19, "characteristics.good": 35595, "characteristics.samples_per_second": 19.34, "characteristics.samples_per_second.normalized_per_core": 0.16116666666666665, "characteristics.samples_per_second.normalized_per_processor": 19.34, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite0-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.5185, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "dfcdf1b10f4962b0", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 338680809, "90.00 percentile latency (ns)": 384612403, "90th percentile latency (ns)": 384612403, "95.00 percentile latency (ns)": 394515685, "97.00 percentile latency (ns)": 399255809, "99.00 percentile latency (ns)": 408530941, "99.90 percentile latency (ns)": 430439075, "Max latency (ns)": 435983293, "Mean latency (ns)": 330610900, "Min duration satisfied": "Yes", "Min latency (ns)": 183702418, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 3.02, "QPS w/o loadgen overhead": 3.02, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 79.268, "characteristics.good": 39634, "characteristics.samples_per_second": 3.02, "characteristics.samples_per_second.normalized_per_core": 0.025166666666666667, "characteristics.samples_per_second.normalized_per_processor": 3.02, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite3-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.0581, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "ab87c2db2474390f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 24694538, "90.00 percentile latency (ns)": 129951774, "90th percentile latency (ns)": 129951774, "95.00 percentile latency (ns)": 134376796, "97.00 percentile latency (ns)": 136962849, "99.00 percentile latency (ns)": 142270004, "99.90 percentile latency (ns)": 151067955, "Max latency (ns)": 163389164, "Mean latency (ns)": 55418062, "Min duration satisfied": "Yes", "Min latency (ns)": 16776661, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 18.04, "QPS w/o loadgen overhead": 18.04, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.216, "characteristics.good": 33108, "characteristics.samples_per_second": 18.04, "characteristics.samples_per_second.normalized_per_core": 0.15033333333333332, "characteristics.samples_per_second.normalized_per_processor": 18.04, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.3846, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "542139e249c03274", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 127614923, "90.00 percentile latency (ns)": 295375440, "90th percentile latency (ns)": 295375440, "95.00 percentile latency (ns)": 312290088, "97.00 percentile latency (ns)": 320011379, "99.00 percentile latency (ns)": 331350400, "99.90 percentile latency (ns)": 355252532, "Max latency (ns)": 374912734, "Mean latency (ns)": 171502423, "Min duration satisfied": "Yes", "Min latency (ns)": 104126544, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 5.83, "QPS w/o loadgen overhead": 5.83, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.344, "characteristics.good": 38172, "characteristics.samples_per_second": 5.83, "characteristics.samples_per_second.normalized_per_core": 0.04858333333333333, "characteristics.samples_per_second.normalized_per_processor": 5.83, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite3-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 6.09756, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "da0d571413c9c217", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7052199, "90.00 percentile latency (ns)": 8092319, "90th percentile latency (ns)": 8092319, "95.00 percentile latency (ns)": 9371529, "97.00 percentile latency (ns)": 14645439, "99.00 percentile latency (ns)": 54385800, "99.90 percentile latency (ns)": 166369944, "Max latency (ns)": 200222067, "Mean latency (ns)": 8549616, "Min duration satisfied": "Yes", "Min latency (ns)": 6038076, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 116.51, "QPS w/o loadgen overhead": 116.96, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 54.568, "characteristics.good": 27284, "characteristics.samples_per_second": 116.96, "characteristics.samples_per_second.normalized_per_core": 0.9746666666666666, "characteristics.samples_per_second.normalized_per_processor": 116.96, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "6416a5112d804b39", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9389939, "90.00 percentile latency (ns)": 27312014, "90th percentile latency (ns)": 27312014, "95.00 percentile latency (ns)": 66791822, "97.00 percentile latency (ns)": 84217551, "99.00 percentile latency (ns)": 95790576, "99.90 percentile latency (ns)": 107371284, "Max latency (ns)": 122592144, "Mean latency (ns)": 15805190, "Min duration satisfied": "Yes", "Min latency (ns)": 8632484, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 63.05, "QPS w/o loadgen overhead": 63.27, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.478, "characteristics.good": 24739, "characteristics.samples_per_second": 63.27, "characteristics.samples_per_second.normalized_per_core": 0.52725, "characteristics.samples_per_second.normalized_per_processor": 63.27, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "9b37c97954ed3b8e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 19217346, "90.00 percentile latency (ns)": 56041496, "90th percentile latency (ns)": 56041496, "95.00 percentile latency (ns)": 101080360, "97.00 percentile latency (ns)": 109438950, "99.00 percentile latency (ns)": 116642578, "99.90 percentile latency (ns)": 126036876, "Max latency (ns)": 136443214, "Mean latency (ns)": 28739025, "Min duration satisfied": "Yes", "Min latency (ns)": 18543304, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 34.71, "QPS w/o loadgen overhead": 34.8, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.126, "characteristics.good": 34063, "characteristics.samples_per_second": 34.8, "characteristics.samples_per_second.normalized_per_core": 0.29, "characteristics.samples_per_second.normalized_per_processor": 34.8, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 43.4783, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "657a18664ae4c98a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 51468921, "90.00 percentile latency (ns)": 185910559, "90th percentile latency (ns)": 185910559, "95.00 percentile latency (ns)": 200095258, "97.00 percentile latency (ns)": 204715220, "99.00 percentile latency (ns)": 213693040, "99.90 percentile latency (ns)": 223655815, "Max latency (ns)": 236151031, "Mean latency (ns)": 83477230, "Min duration satisfied": "Yes", "Min latency (ns)": 46050639, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 11.98, "QPS w/o loadgen overhead": 11.98, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.76, "characteristics.good": 37880, "characteristics.samples_per_second": 11.98, "characteristics.samples_per_second.normalized_per_core": 0.09983333333333334, "characteristics.samples_per_second.normalized_per_processor": 11.98, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 13.3333, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "230b52a51e0c4587", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 42534055, "90.00 percentile latency (ns)": 186664897, "90th percentile latency (ns)": 186664897, "95.00 percentile latency (ns)": 193130920, "97.00 percentile latency (ns)": 197007772, "99.00 percentile latency (ns)": 203216855, "99.90 percentile latency (ns)": 214667317, "Max latency (ns)": 226096280, "Mean latency (ns)": 78798543, "Min duration satisfied": "Yes", "Min latency (ns)": 25397175, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 12.68, "QPS w/o loadgen overhead": 12.69, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 65.75, "characteristics.good": 32875, "characteristics.samples_per_second": 12.69, "characteristics.samples_per_second.normalized_per_core": 0.10575, "characteristics.samples_per_second.normalized_per_processor": 12.69, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.2857, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "a4f83af4046a3903", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11468668, "90.00 percentile latency (ns)": 14440978, "90th percentile latency (ns)": 14440978, "95.00 percentile latency (ns)": 24735630, "97.00 percentile latency (ns)": 41381753, "99.00 percentile latency (ns)": 93226260, "99.90 percentile latency (ns)": 111032272, "Max latency (ns)": 121044296, "Mean latency (ns)": 14277895, "Min duration satisfied": "Yes", "Min latency (ns)": 10718798, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 69.84, "QPS w/o loadgen overhead": 70.04, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.768, "characteristics.good": 32384, "characteristics.samples_per_second": 70.04, "characteristics.samples_per_second.normalized_per_core": 0.5836666666666667, "characteristics.samples_per_second.normalized_per_processor": 70.04, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "03af474fff8915ab", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 46440958, "90.00 percentile latency (ns)": 169783095, "90th percentile latency (ns)": 169783095, "95.00 percentile latency (ns)": 198553376, "97.00 percentile latency (ns)": 205144817, "99.00 percentile latency (ns)": 212489210, "99.90 percentile latency (ns)": 225700565, "Max latency (ns)": 238968028, "Mean latency (ns)": 69951840, "Min duration satisfied": "Yes", "Min latency (ns)": 40195358, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 14.29, "QPS w/o loadgen overhead": 14.3, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.122, "characteristics.good": 35061, "characteristics.samples_per_second": 14.3, "characteristics.samples_per_second.normalized_per_core": 0.11916666666666667, "characteristics.samples_per_second.normalized_per_processor": 14.3, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.7059, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "a7838a65845829c5", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11225102, "90.00 percentile latency (ns)": 13882770, "90th percentile latency (ns)": 13882770, "95.00 percentile latency (ns)": 23419365, "97.00 percentile latency (ns)": 37345636, "99.00 percentile latency (ns)": 90020162, "99.90 percentile latency (ns)": 110272613, "Max latency (ns)": 129847445, "Mean latency (ns)": 13763619, "Min duration satisfied": "Yes", "Min latency (ns)": 10297306, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 72.46, "QPS w/o loadgen overhead": 72.66, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.08, "characteristics.good": 32040, "characteristics.samples_per_second": 72.66, "characteristics.samples_per_second.normalized_per_core": 0.6054999999999999, "characteristics.samples_per_second.normalized_per_processor": 72.66, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "5a477f461c94eb1d", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20367954, "90.00 percentile latency (ns)": 148922073, "90th percentile latency (ns)": 148922073, "95.00 percentile latency (ns)": 179249210, "97.00 percentile latency (ns)": 185605834, "99.00 percentile latency (ns)": 194590426, "99.90 percentile latency (ns)": 207254988, "Max latency (ns)": 221414826, "Mean latency (ns)": 46216769, "Min duration satisfied": "Yes", "Min latency (ns)": 18826787, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 21.62, "QPS w/o loadgen overhead": 21.64, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.282, "characteristics.good": 32141, "characteristics.samples_per_second": 21.64, "characteristics.samples_per_second.normalized_per_core": 0.18033333333333335, "characteristics.samples_per_second.normalized_per_processor": 21.64, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 25.641, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "9158bc4813950911", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 45663323, "90.00 percentile latency (ns)": 179931655, "90th percentile latency (ns)": 179931655, "95.00 percentile latency (ns)": 190818048, "97.00 percentile latency (ns)": 195054529, "99.00 percentile latency (ns)": 201582608, "99.90 percentile latency (ns)": 215828415, "Max latency (ns)": 228741940, "Mean latency (ns)": 74988155, "Min duration satisfied": "Yes", "Min latency (ns)": 37700791, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 13.33, "QPS w/o loadgen overhead": 13.34, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.678, "characteristics.good": 36339, "characteristics.samples_per_second": 13.34, "characteristics.samples_per_second.normalized_per_core": 0.11116666666666666, "characteristics.samples_per_second.normalized_per_processor": 13.34, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 13.5135, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "36a1a18035db1688", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 13472327, "90.00 percentile latency (ns)": 16677745, "90th percentile latency (ns)": 16677745, "95.00 percentile latency (ns)": 31798268, "97.00 percentile latency (ns)": 50458982, "99.00 percentile latency (ns)": 150205507, "99.90 percentile latency (ns)": 193721578, "Max latency (ns)": 214138863, "Mean latency (ns)": 17412362, "Min duration satisfied": "Yes", "Min latency (ns)": 12094214, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 57.29, "QPS w/o loadgen overhead": 57.43, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.972, "characteristics.good": 32486, "characteristics.samples_per_second": 57.43, "characteristics.samples_per_second.normalized_per_core": 0.4785833333333333, "characteristics.samples_per_second.normalized_per_processor": 57.43, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 41.6667, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "f10cad9518461085", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 59963586, "90.00 percentile latency (ns)": 182029805, "90th percentile latency (ns)": 182029805, "95.00 percentile latency (ns)": 211403045, "97.00 percentile latency (ns)": 217823881, "99.00 percentile latency (ns)": 227027353, "99.90 percentile latency (ns)": 242318174, "Max latency (ns)": 253149982, "Mean latency (ns)": 81394957, "Min duration satisfied": "Yes", "Min latency (ns)": 50252138, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 12.28, "QPS w/o loadgen overhead": 12.29, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 71.77, "characteristics.good": 35885, "characteristics.samples_per_second": 12.29, "characteristics.samples_per_second.normalized_per_core": 0.10241666666666666, "characteristics.samples_per_second.normalized_per_processor": 12.29, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 11.4943, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "4c4fa4a7d59f7c5e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 30073037, "90.00 percentile latency (ns)": 182199877, "90th percentile latency (ns)": 182199877, "95.00 percentile latency (ns)": 191283729, "97.00 percentile latency (ns)": 194947517, "99.00 percentile latency (ns)": 202214967, "99.90 percentile latency (ns)": 218102169, "Max latency (ns)": 230073624, "Mean latency (ns)": 66258422, "Min duration satisfied": "Yes", "Min latency (ns)": 22349261, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 15.08, "QPS w/o loadgen overhead": 15.09, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 66.75, "characteristics.good": 33375, "characteristics.samples_per_second": 15.09, "characteristics.samples_per_second.normalized_per_core": 0.12575, "characteristics.samples_per_second.normalized_per_processor": 15.09, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 21.2766, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "0a14d2192bcb92d8", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10105665, "90.00 percentile latency (ns)": 91087770, "90th percentile latency (ns)": 91087770, "95.00 percentile latency (ns)": 121333015, "97.00 percentile latency (ns)": 134812678, "99.00 percentile latency (ns)": 150460298, "99.90 percentile latency (ns)": 169069212, "Max latency (ns)": 179874001, "Mean latency (ns)": 30959111, "Min duration satisfied": "Yes", "Min latency (ns)": 6209874, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 32.25, "QPS w/o loadgen overhead": 32.3, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 47.14, "characteristics.good": 23570, "characteristics.samples_per_second": 32.3, "characteristics.samples_per_second.normalized_per_core": 0.26916666666666667, "characteristics.samples_per_second.normalized_per_processor": 32.3, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "78b54182b252a927", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 17559204, "90.00 percentile latency (ns)": 156599541, "90th percentile latency (ns)": 156599541, "95.00 percentile latency (ns)": 169983837, "97.00 percentile latency (ns)": 175483555, "99.00 percentile latency (ns)": 182867926, "99.90 percentile latency (ns)": 195320377, "Max latency (ns)": 210273243, "Mean latency (ns)": 49918524, "Min duration satisfied": "Yes", "Min latency (ns)": 11373830, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.01, "QPS w/o loadgen overhead": 20.03, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.988, "characteristics.good": 28494, "characteristics.samples_per_second": 20.03, "characteristics.samples_per_second.normalized_per_core": 0.16691666666666669, "characteristics.samples_per_second.normalized_per_processor": 20.03, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.5185, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "2fad4e429c8e871a", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 36975938, "90.00 percentile latency (ns)": 108743825, "90th percentile latency (ns)": 108743825, "95.00 percentile latency (ns)": 112301012, "97.00 percentile latency (ns)": 114381960, "99.00 percentile latency (ns)": 118896112, "99.90 percentile latency (ns)": 127578834, "Max latency (ns)": 133596411, "Mean latency (ns)": 54781546, "Min duration satisfied": "Yes", "Min latency (ns)": 15670050, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 18.23, "QPS w/o loadgen overhead": 18.25, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 63.716, "characteristics.good": 31858, "characteristics.samples_per_second": 18.25, "characteristics.samples_per_second.normalized_per_core": 0.15208333333333332, "characteristics.samples_per_second.normalized_per_processor": 18.25, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 17.2414, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "f19f9aab5293cd2d", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6556617, "90.00 percentile latency (ns)": 7594362, "90th percentile latency (ns)": 7594362, "95.00 percentile latency (ns)": 8855246, "97.00 percentile latency (ns)": 13378108, "99.00 percentile latency (ns)": 49683963, "99.90 percentile latency (ns)": 169919574, "Max latency (ns)": 194789498, "Mean latency (ns)": 8104579, "Min duration satisfied": "Yes", "Min latency (ns)": 5472514, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 122.88, "QPS w/o loadgen overhead": 123.39, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 55.54, "characteristics.good": 27770, "characteristics.samples_per_second": 123.39, "characteristics.samples_per_second.normalized_per_core": 1.02825, "characteristics.samples_per_second.normalized_per_processor": 123.39, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "2234a74e3daf0ad0", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20976984, "90.00 percentile latency (ns)": 165854126, "90th percentile latency (ns)": 165854126, "95.00 percentile latency (ns)": 177221954, "97.00 percentile latency (ns)": 182063616, "99.00 percentile latency (ns)": 190492528, "99.90 percentile latency (ns)": 204728632, "Max latency (ns)": 221266486, "Mean latency (ns)": 55258917, "Min duration satisfied": "Yes", "Min latency (ns)": 14844023, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 18.08, "QPS w/o loadgen overhead": 18.1, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.322, "characteristics.good": 29661, "characteristics.samples_per_second": 18.1, "characteristics.samples_per_second.normalized_per_core": 0.15083333333333335, "characteristics.samples_per_second.normalized_per_processor": 18.1, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 120, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/open/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.625, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "8e9744571b7b5a93", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 413086676, "90.00 percentile latency (ns)": 414838695, "90th percentile latency (ns)": 414838695, "95.00 percentile latency (ns)": 415286343, "97.00 percentile latency (ns)": 415812713, "99.00 percentile latency (ns)": 416908249, "99.90 percentile latency (ns)": 425225132, "Max latency (ns)": 426155625, "Mean latency (ns)": 412730167, "Min duration satisfied": "Yes", "Min latency (ns)": 403893708, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.42, "QPS w/o loadgen overhead": 2.42, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.samples_per_second": 2.42, "characteristics.samples_per_second.normalized_per_core": 0.4033333333333333, "characteristics.samples_per_second.normalized_per_processor": 2.42, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-neon", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/Krai/results/firefly-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-neon", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.5, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "3b48fd3de22a04ad", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 589167631, "90.00 percentile latency (ns)": 603680875, "90th percentile latency (ns)": 603680875, "95.00 percentile latency (ns)": 607960147, "97.00 percentile latency (ns)": 611826996, "99.00 percentile latency (ns)": 661061671, "99.90 percentile latency (ns)": 714376068, "Max latency (ns)": 722865773, "Mean latency (ns)": 590196558, "Min duration satisfied": "Yes", "Min latency (ns)": 569764409, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.69, "QPS w/o loadgen overhead": 1.69, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.power": 5.834231404958682, "characteristics.power.normalized_per_core": 0.04861859504132235, "characteristics.power.normalized_per_processor": 5.834231404958682, "characteristics.samples_per_second": 1.69, "characteristics.samples_per_second.normalized_per_core": 0.014083333333333333, "characteristics.samples_per_second.normalized_per_processor": 1.69, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.off-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan off", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 120, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 120, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/Krai/results/rpi4coral-fan.off-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.off-tflite-v2.4.1-ruy", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 1.81818, "task": "image classification", "task2": "image classification", "total_cores": 120, "uid": "8e8c3ff4c998eadc", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite" }, { "50.00 percentile latency (ns)": 554806855, "90.00 percentile latency (ns)": 555966181, "90th percentile latency (ns)": 555966181, "95.00 percentile latency (ns)": 556913577, "97.00 percentile latency (ns)": 558372138, "99.00 percentile latency (ns)": 563261714, "99.90 percentile latency (ns)": 578372854, "Max latency (ns)": 581154408, "Mean latency (ns)": 555006161, "Min duration satisfied": "Yes", "Min latency (ns)": 543793798, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.8, "QPS w/o loadgen overhead": 1.8, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "800 MHz", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "4 GiB (shared with host)", "accelerator_memory_configuration": "-", "accelerator_model_name": "Arm Mali-T860 MP4", "accelerator_on-chip_memories": "-", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.samples_per_second": 1.8, "characteristics.samples_per_second.normalized_per_core": 1.8, "characteristics.samples_per_second.normalized_per_processor": 1.8, "characteristics.total": 50000, "ck_system": "firefly-armnn-v21.02-opencl", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (OpenCL)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/Krai/results/firefly-armnn-v21.02-opencl", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-armnn-v21.02-opencl", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.5, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "8e9ef249f7b55579", "use_accelerator": true, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 49760932, "90.00 percentile latency (ns)": 68502340, "90th percentile latency (ns)": 68502340, "95.00 percentile latency (ns)": 78511161, "97.00 percentile latency (ns)": 86730642, "99.00 percentile latency (ns)": 96095375, "99.90 percentile latency (ns)": 108083443, "Max latency (ns)": 120593298, "Mean latency (ns)": 55518487, "Min duration satisfied": "Yes", "Min latency (ns)": 47022883, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 18.01, "QPS w/o loadgen overhead": 18.01, "Result is": "VALID", "SUT name": "ArmNN_TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.power": 23.160516666666673, "characteristics.power.normalized_per_core": 2.895064583333334, "characteristics.power.normalized_per_processor": 23.160516666666673, "characteristics.samples_per_second": 18.01, "characteristics.samples_per_second.normalized_per_core": 2.25125, "characteristics.samples_per_second.normalized_per_processor": 18.01, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-armnn-v21.02-neon", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "ArmNN v21.02 (Neon)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/Krai/results/xavier-power.0-fan.cool-armnn-v21.02-neon", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.2", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-armnn-v21.02-neon", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 20, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "5824a71da2374363", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite -> ArmNN" }, { "50.00 percentile latency (ns)": 75739770, "90.00 percentile latency (ns)": 77975401, "90th percentile latency (ns)": 77975401, "95.00 percentile latency (ns)": 88838337, "97.00 percentile latency (ns)": 90892308, "99.00 percentile latency (ns)": 100776825, "99.90 percentile latency (ns)": 142315902, "Max latency (ns)": 162143685, "Mean latency (ns)": 77278132, "Min duration satisfied": "Yes", "Min latency (ns)": 73726788, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 12.94, "QPS w/o loadgen overhead": 12.94, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.power": 28.172995008319464, "characteristics.power.normalized_per_core": 3.521624376039933, "characteristics.power.normalized_per_processor": 28.172995008319464, "characteristics.samples_per_second": 12.94, "characteristics.samples_per_second.normalized_per_core": 1.6175, "characteristics.samples_per_second.normalized_per_processor": 12.94, "characteristics.total": 50000, "ck_system": "xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "Fan mode: cool", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 8, "host_processor_frequency": "2265.6 MHz", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "400 GB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "Power mode: MAXN", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 8, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/Krai/results/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.2", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/xavier-power.0-fan.cool-tflite-v2.4.1-ruy", "system_name": "NVIDIA Jetson AGX Xavier", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 11.3636, "task": "image classification", "task2": "image classification", "total_cores": 8, "uid": "4fbc867f629b121e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite" }, { "50.00 percentile latency (ns)": 591282177, "90.00 percentile latency (ns)": 606294547, "90th percentile latency (ns)": 606294547, "95.00 percentile latency (ns)": 611230567, "97.00 percentile latency (ns)": 613858717, "99.00 percentile latency (ns)": 646968602, "99.90 percentile latency (ns)": 727313337, "Max latency (ns)": 729805258, "Mean latency (ns)": 592611844, "Min duration satisfied": "Yes", "Min latency (ns)": 570831810, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.69, "QPS w/o loadgen overhead": 1.69, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.power": 6.6994563426688645, "characteristics.power.normalized_per_core": 1.6748640856672161, "characteristics.power.normalized_per_processor": 6.6994563426688645, "characteristics.samples_per_second": 1.69, "characteristics.samples_per_second.normalized_per_core": 0.4225, "characteristics.samples_per_second.normalized_per_processor": 1.69, "characteristics.total": 50000, "ck_system": "rpi4coral-fan.on-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "http://www.raspberrypiwiki.com/index.php/Armor_Case_B: fan on", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 128=4x32 KiB, L1D$ 128=4x32 KiB, L2$ 1 MiB", "host_processor_core_count": 4, "host_processor_frequency": "1500 MHz", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP4", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme Pro microSD", "hw_notes": "", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 4, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/Krai/results/rpi4coral-fan.on-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 20.04.2; kernel 5.4.0-1032-raspi #35-Ubuntu SMP PREEMPT (Fri Mar 19 20:52:40 UTC 2021)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge 1.55.2", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/rpi4coral-fan.on-tflite-v2.4.1-ruy", "system_name": "Raspberry Pi 4", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 1.81818, "task": "image classification", "task2": "image classification", "total_cores": 4, "uid": "98aadc5b124960b9", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite" }, { "50.00 percentile latency (ns)": 450754388, "90.00 percentile latency (ns)": 509104821, "90th percentile latency (ns)": 509104821, "95.00 percentile latency (ns)": 550684956, "97.00 percentile latency (ns)": 583679140, "99.00 percentile latency (ns)": 603389125, "99.90 percentile latency (ns)": 649859307, "Max latency (ns)": 676381492, "Mean latency (ns)": 461404812, "Min duration satisfied": "Yes", "Min latency (ns)": 409320513, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.17, "QPS w/o loadgen overhead": 2.17, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.samples_per_second": 2.17, "characteristics.samples_per_second.normalized_per_core": 0.36166666666666664, "characteristics.samples_per_second.normalized_per_processor": 2.17, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.4.1-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/mlcommons/ck-mlops/tree/main/package", "framework": "TFLite v2.4.1 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "normalize_cores": 6, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/Krai/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/Krai/results/firefly-tflite-v2.4.1-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 7322528924094909334, "retraining": "no", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.4.1-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 1.83486, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "a748ee09cb996f70", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite" } ]