[ { "50.00 percentile latency (ns)": 74152147, "90.00 percentile latency (ns)": 149194463, "90th percentile latency (ns)": 149194463, "95.00 percentile latency (ns)": 196076656, "97.00 percentile latency (ns)": 205041062, "99.00 percentile latency (ns)": 214697846, "99.90 percentile latency (ns)": 223914862, "Max latency (ns)": 228916589, "Mean latency (ns)": 87843985, "Min duration satisfied": "Yes", "Min latency (ns)": 63649237, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 11.38, "QPS w/o loadgen overhead": 11.38, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 149.194463, "characteristics.90th_percentile_latency_ns": 149194463.0, "characteristics.90th_percentile_latency_s": 0.149194463, "characteristics.90th_percentile_latency_us": 149194.463, "characteristics.accuracy": 75.068, "characteristics.good": 37534, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite0-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 10.8696, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "5b605a28a478dbe0", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 37430736, "90.00 percentile latency (ns)": 76605925, "90th percentile latency (ns)": 76605925, "95.00 percentile latency (ns)": 139227907, "97.00 percentile latency (ns)": 168307146, "99.00 percentile latency (ns)": 186914303, "99.90 percentile latency (ns)": 201449122, "Max latency (ns)": 213005614, "Mean latency (ns)": 49984386, "Min duration satisfied": "Yes", "Min latency (ns)": 34854737, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 19.98, "QPS w/o loadgen overhead": 20.01, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 76.605925, "characteristics.90th_percentile_latency_ns": 76605925.0, "characteristics.90th_percentile_latency_s": 0.076605925, "characteristics.90th_percentile_latency_us": 76605.925, "characteristics.accuracy": 71.19, "characteristics.good": 35595, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite0-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite0-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite0.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.5185, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "d002e04afea05404", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 106646431, "90.00 percentile latency (ns)": 188719818, "90th percentile latency (ns)": 188719818, "95.00 percentile latency (ns)": 251657516, "97.00 percentile latency (ns)": 269446840, "99.00 percentile latency (ns)": 285280972, "99.90 percentile latency (ns)": 298914223, "Max latency (ns)": 318689039, "Mean latency (ns)": 122777928, "Min duration satisfied": "Yes", "Min latency (ns)": 92435266, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 8.14, "QPS w/o loadgen overhead": 8.14, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 188.719818, "characteristics.90th_percentile_latency_ns": 188719818.0, "characteristics.90th_percentile_latency_s": 0.188719818, "characteristics.90th_percentile_latency_us": 188719.818, "characteristics.accuracy": 76.642, "characteristics.good": 38321, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite1-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 7.69231, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "33e49749bf4d3c7b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 58536162, "90.00 percentile latency (ns)": 100737892, "90th percentile latency (ns)": 100737892, "95.00 percentile latency (ns)": 168624029, "97.00 percentile latency (ns)": 212149782, "99.00 percentile latency (ns)": 242464318, "99.90 percentile latency (ns)": 265273246, "Max latency (ns)": 282413735, "Mean latency (ns)": 72438353, "Min duration satisfied": "Yes", "Min latency (ns)": 53482196, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 13.8, "QPS w/o loadgen overhead": 13.8, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 100.737892, "characteristics.90th_percentile_latency_ns": 100737892.0, "characteristics.90th_percentile_latency_s": 0.100737892, "characteristics.90th_percentile_latency_us": 100737.892, "characteristics.accuracy": 74.098, "characteristics.good": 37049, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite1-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.987, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "bac22c3dcc412689", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 235354940, "90.00 percentile latency (ns)": 293672880, "90th percentile latency (ns)": 293672880, "95.00 percentile latency (ns)": 303602443, "97.00 percentile latency (ns)": 309512456, "99.00 percentile latency (ns)": 317951459, "99.90 percentile latency (ns)": 341667413, "Max latency (ns)": 448119929, "Mean latency (ns)": 226369438, "Min duration satisfied": "Yes", "Min latency (ns)": 123688929, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 4.42, "QPS w/o loadgen overhead": 4.42, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 293.67288, "characteristics.90th_percentile_latency_ns": 293672880.0, "characteristics.90th_percentile_latency_s": 0.29367288, "characteristics.90th_percentile_latency_us": 293672.88, "characteristics.accuracy": 77.542, "characteristics.good": 38771, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite2-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 4.1841, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "6764d3840308a70c", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 82084321, "90.00 percentile latency (ns)": 218946291, "90th percentile latency (ns)": 218946291, "95.00 percentile latency (ns)": 240552618, "97.00 percentile latency (ns)": 253173361, "99.00 percentile latency (ns)": 264615158, "99.90 percentile latency (ns)": 280148174, "Max latency (ns)": 299587688, "Mean latency (ns)": 115944873, "Min duration satisfied": "Yes", "Min latency (ns)": 71941036, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 8.62, "QPS w/o loadgen overhead": 8.62, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 218.946291, "characteristics.90th_percentile_latency_ns": 218946291.0, "characteristics.90th_percentile_latency_s": 0.218946291, "characteristics.90th_percentile_latency_us": 218946.291, "characteristics.accuracy": 74.124, "characteristics.good": 37062, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite2-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 9.25926, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "7defeab6f7804f22", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 328586430, "90.00 percentile latency (ns)": 380736463, "90th percentile latency (ns)": 380736463, "95.00 percentile latency (ns)": 390845357, "97.00 percentile latency (ns)": 396542383, "99.00 percentile latency (ns)": 409008172, "99.90 percentile latency (ns)": 430273817, "Max latency (ns)": 436923714, "Mean latency (ns)": 324454976, "Min duration satisfied": "Yes", "Min latency (ns)": 183151052, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 3.08, "QPS w/o loadgen overhead": 3.08, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 380.736463, "characteristics.90th_percentile_latency_ns": 380736463.0, "characteristics.90th_percentile_latency_s": 0.380736463, "characteristics.90th_percentile_latency_us": 380736.463, "characteristics.accuracy": 79.268, "characteristics.good": 39634, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite3-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.0581, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "e46f989cf79fc446", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 115874378, "90.00 percentile latency (ns)": 269592209, "90th percentile latency (ns)": 269592209, "95.00 percentile latency (ns)": 300265842, "97.00 percentile latency (ns)": 312628754, "99.00 percentile latency (ns)": 325390025, "99.90 percentile latency (ns)": 345693816, "Max latency (ns)": 365999025, "Mean latency (ns)": 150988597, "Min duration satisfied": "Yes", "Min latency (ns)": 104186136, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 6.62, "QPS w/o loadgen overhead": 6.62, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 269.592209, "characteristics.90th_percentile_latency_ns": 269592209.0, "characteristics.90th_percentile_latency_s": 0.269592209, "characteristics.90th_percentile_latency_us": 269592.209, "characteristics.accuracy": 76.344, "characteristics.good": 38172, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite3-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 6.09756, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "4e89a1209995f834", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 493166225, "90.00 percentile latency (ns)": 545062465, "90th percentile latency (ns)": 545062465, "95.00 percentile latency (ns)": 558331316, "97.00 percentile latency (ns)": 563402405, "99.00 percentile latency (ns)": 578899309, "99.90 percentile latency (ns)": 603919584, "Max latency (ns)": 606813127, "Mean latency (ns)": 486407759, "Min duration satisfied": "Yes", "Min latency (ns)": 317799355, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.06, "QPS w/o loadgen overhead": 2.06, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 545.062465, "characteristics.90th_percentile_latency_ns": 545062465.0, "characteristics.90th_percentile_latency_s": 0.545062465, "characteristics.90th_percentile_latency_us": 545062.465, "characteristics.accuracy": 80.676, "characteristics.good": 40338, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite4-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.04082, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "ee98101daf54b143", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 324260412, "90.00 percentile latency (ns)": 416209128, "90th percentile latency (ns)": 416209128, "95.00 percentile latency (ns)": 433856470, "97.00 percentile latency (ns)": 440011178, "99.00 percentile latency (ns)": 455279870, "99.90 percentile latency (ns)": 473264667, "Max latency (ns)": 476895076, "Mean latency (ns)": 305669961, "Min duration satisfied": "Yes", "Min latency (ns)": 178695293, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 3.27, "QPS w/o loadgen overhead": 3.27, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 416.209128, "characteristics.90th_percentile_latency_ns": 416209128.0, "characteristics.90th_percentile_latency_s": 0.416209128, "characteristics.90th_percentile_latency_us": 416209.128, "characteristics.accuracy": 78.514, "characteristics.good": 39257, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite4-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.3557, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "d668e9178b37aa61", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4868214, "90.00 percentile latency (ns)": 35494840, "90th percentile latency (ns)": 35494840, "95.00 percentile latency (ns)": 64486175, "97.00 percentile latency (ns)": 76170783, "99.00 percentile latency (ns)": 85462701, "99.90 percentile latency (ns)": 96201139, "Max latency (ns)": 108830150, "Mean latency (ns)": 12620822, "Min duration satisfied": "Yes", "Min latency (ns)": 4572172, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 78.93, "QPS w/o loadgen overhead": 79.23, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 35.49484, "characteristics.90th_percentile_latency_ns": 35494840.0, "characteristics.90th_percentile_latency_s": 0.03549484, "characteristics.90th_percentile_latency_us": 35494.84, "characteristics.accuracy": 44.116, "characteristics.good": 22058, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "ed700701d4c0baa6", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 2730588, "90.00 percentile latency (ns)": 2926880, "90th percentile latency (ns)": 2926880, "95.00 percentile latency (ns)": 3304880, "97.00 percentile latency (ns)": 4201173, "99.00 percentile latency (ns)": 10234308, "99.90 percentile latency (ns)": 76645992, "Max latency (ns)": 97319692, "Mean latency (ns)": 3024682, "Min duration satisfied": "Yes", "Min latency (ns)": 2192462, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 326.77, "QPS w/o loadgen overhead": 330.61, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 2.92688, "characteristics.90th_percentile_latency_ns": 2926880.0, "characteristics.90th_percentile_latency_s": 0.00292688, "characteristics.90th_percentile_latency_us": 2926.88, "characteristics.accuracy": 42.384, "characteristics.good": 21192, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "d7f0556a46b9b8f5", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10213590, "90.00 percentile latency (ns)": 78457510, "90th percentile latency (ns)": 78457510, "95.00 percentile latency (ns)": 87635098, "97.00 percentile latency (ns)": 90668724, "99.00 percentile latency (ns)": 95825396, "99.90 percentile latency (ns)": 103955319, "Max latency (ns)": 116438366, "Mean latency (ns)": 25448126, "Min duration satisfied": "Yes", "Min latency (ns)": 6526921, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 39.2, "QPS w/o loadgen overhead": 39.3, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 78.45751, "characteristics.90th_percentile_latency_ns": 78457510.0, "characteristics.90th_percentile_latency_s": 0.07845751, "characteristics.90th_percentile_latency_us": 78457.51, "characteristics.accuracy": 47.808, "characteristics.good": 23904, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 37.037, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "f08ba16710b91c23", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3958506, "90.00 percentile latency (ns)": 4203799, "90th percentile latency (ns)": 4203799, "95.00 percentile latency (ns)": 5117884, "97.00 percentile latency (ns)": 5860760, "99.00 percentile latency (ns)": 20880161, "99.90 percentile latency (ns)": 82237296, "Max latency (ns)": 105842499, "Mean latency (ns)": 4395911, "Min duration satisfied": "Yes", "Min latency (ns)": 3143880, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 225.98, "QPS w/o loadgen overhead": 227.48, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 4.203799, "characteristics.90th_percentile_latency_ns": 4203799.0, "characteristics.90th_percentile_latency_s": 0.004203799, "characteristics.90th_percentile_latency_us": 4203.799, "characteristics.accuracy": 45.494, "characteristics.good": 22747, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "4c541e6ae1278649", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9956947, "90.00 percentile latency (ns)": 22275246, "90th percentile latency (ns)": 22275246, "95.00 percentile latency (ns)": 45031125, "97.00 percentile latency (ns)": 67555777, "99.00 percentile latency (ns)": 89787574, "99.90 percentile latency (ns)": 100707019, "Max latency (ns)": 112317832, "Mean latency (ns)": 14560577, "Min duration satisfied": "Yes", "Min latency (ns)": 8737199, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 68.47, "QPS w/o loadgen overhead": 68.68, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 22.275246, "characteristics.90th_percentile_latency_ns": 22275246.0, "characteristics.90th_percentile_latency_s": 0.022275246, "characteristics.90th_percentile_latency_us": 22275.246, "characteristics.accuracy": 49.478, "characteristics.good": 24739, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "64f6e9e85d89813b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4947540, "90.00 percentile latency (ns)": 5219082, "90th percentile latency (ns)": 5219082, "95.00 percentile latency (ns)": 6182164, "97.00 percentile latency (ns)": 7044331, "99.00 percentile latency (ns)": 29754655, "99.90 percentile latency (ns)": 92626258, "Max latency (ns)": 107109913, "Mean latency (ns)": 5504405, "Min duration satisfied": "Yes", "Min latency (ns)": 3904832, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 180.71, "QPS w/o loadgen overhead": 181.67, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 5.219082, "characteristics.90th_percentile_latency_ns": 5219082.0, "characteristics.90th_percentile_latency_s": 0.005219082, "characteristics.90th_percentile_latency_us": 5219.082, "characteristics.accuracy": 48.162, "characteristics.good": 24081, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "f3fa98f233a11ff9", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 15285732, "90.00 percentile latency (ns)": 66472346, "90th percentile latency (ns)": 66472346, "95.00 percentile latency (ns)": 90631699, "97.00 percentile latency (ns)": 95329269, "99.00 percentile latency (ns)": 100950435, "99.90 percentile latency (ns)": 110517228, "Max latency (ns)": 127731957, "Mean latency (ns)": 26353731, "Min duration satisfied": "Yes", "Min latency (ns)": 11648032, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 37.88, "QPS w/o loadgen overhead": 37.95, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 66.472346, "characteristics.90th_percentile_latency_ns": 66472346.0, "characteristics.90th_percentile_latency_s": 0.066472346, "characteristics.90th_percentile_latency_us": 66472.346, "characteristics.accuracy": 51.848, "characteristics.good": 25924, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 21.2766, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "c82f94a0daee99d0", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5445188, "90.00 percentile latency (ns)": 6685614, "90th percentile latency (ns)": 6685614, "95.00 percentile latency (ns)": 9236543, "97.00 percentile latency (ns)": 22088027, "99.00 percentile latency (ns)": 80195437, "99.90 percentile latency (ns)": 98101453, "Max latency (ns)": 110506728, "Mean latency (ns)": 7363468, "Min duration satisfied": "Yes", "Min latency (ns)": 4988116, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 135.23, "QPS w/o loadgen overhead": 135.81, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 6.685614, "characteristics.90th_percentile_latency_ns": 6685614.0, "characteristics.90th_percentile_latency_s": 0.006685614, "characteristics.90th_percentile_latency_us": 6685.614, "characteristics.accuracy": 50.242, "characteristics.good": 25121, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "6423d04099be3c75", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10253257, "90.00 percentile latency (ns)": 78455495, "90th percentile latency (ns)": 78455495, "95.00 percentile latency (ns)": 93172117, "97.00 percentile latency (ns)": 96517047, "99.00 percentile latency (ns)": 101052684, "99.90 percentile latency (ns)": 108020001, "Max latency (ns)": 115166725, "Mean latency (ns)": 23420636, "Min duration satisfied": "Yes", "Min latency (ns)": 8704214, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 42.58, "QPS w/o loadgen overhead": 42.7, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 78.455495, "characteristics.90th_percentile_latency_ns": 78455495.0, "characteristics.90th_percentile_latency_s": 0.078455495, "characteristics.90th_percentile_latency_us": 78455.495, "characteristics.accuracy": 57.918, "characteristics.good": 28959, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "3a7aff5613921ca5", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4758237, "90.00 percentile latency (ns)": 5070901, "90th percentile latency (ns)": 5070901, "95.00 percentile latency (ns)": 6425106, "97.00 percentile latency (ns)": 7324606, "99.00 percentile latency (ns)": 30748539, "99.90 percentile latency (ns)": 92909809, "Max latency (ns)": 110418255, "Mean latency (ns)": 5556298, "Min duration satisfied": "Yes", "Min latency (ns)": 4188612, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 178.91, "QPS w/o loadgen overhead": 179.98, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 5.070901, "characteristics.90th_percentile_latency_ns": 5070901.0, "characteristics.90th_percentile_latency_s": 0.005070901, "characteristics.90th_percentile_latency_us": 5070.901, "characteristics.accuracy": 56.75, "characteristics.good": 28375, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "0a44d8fb87a10ddf", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 19889070, "90.00 percentile latency (ns)": 100591482, "90th percentile latency (ns)": 100591482, "95.00 percentile latency (ns)": 104656294, "97.00 percentile latency (ns)": 106799125, "99.00 percentile latency (ns)": 110147798, "99.90 percentile latency (ns)": 116380992, "Max latency (ns)": 121208672, "Mean latency (ns)": 40383670, "Min duration satisfied": "Yes", "Min latency (ns)": 12801268, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 24.73, "QPS w/o loadgen overhead": 24.76, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 100.591482, "characteristics.90th_percentile_latency_ns": 100591482.0, "characteristics.90th_percentile_latency_s": 0.100591482, "characteristics.90th_percentile_latency_us": 100591.482, "characteristics.accuracy": 60.842, "characteristics.good": 30421, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.8679, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "4ec73448cb3de01a", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6887138, "90.00 percentile latency (ns)": 7588599, "90th percentile latency (ns)": 7588599, "95.00 percentile latency (ns)": 9032935, "97.00 percentile latency (ns)": 13319692, "99.00 percentile latency (ns)": 45967515, "99.90 percentile latency (ns)": 99540194, "Max latency (ns)": 111103213, "Mean latency (ns)": 8016714, "Min duration satisfied": "Yes", "Min latency (ns)": 6044803, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 124.25, "QPS w/o loadgen overhead": 124.74, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 7.588599, "characteristics.90th_percentile_latency_ns": 7588599.0, "characteristics.90th_percentile_latency_s": 0.007588599, "characteristics.90th_percentile_latency_us": 7588.599, "characteristics.accuracy": 59.444, "characteristics.good": 29722, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "2176c0463f8aed1e", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 24845427, "90.00 percentile latency (ns)": 106397406, "90th percentile latency (ns)": 106397406, "95.00 percentile latency (ns)": 110835831, "97.00 percentile latency (ns)": 113210040, "99.00 percentile latency (ns)": 118383452, "99.90 percentile latency (ns)": 126807003, "Max latency (ns)": 133033629, "Mean latency (ns)": 46232417, "Min duration satisfied": "Yes", "Min latency (ns)": 16990535, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 21.6, "QPS w/o loadgen overhead": 21.63, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 106.397406, "characteristics.90th_percentile_latency_ns": 106397406.0, "characteristics.90th_percentile_latency_s": 0.106397406, "characteristics.90th_percentile_latency_us": 106397.406, "characteristics.accuracy": 63.246, "characteristics.good": 31623, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 16.9492, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "20f710b1cf5d49e6", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8911875, "90.00 percentile latency (ns)": 9939708, "90th percentile latency (ns)": 9939708, "95.00 percentile latency (ns)": 12300745, "97.00 percentile latency (ns)": 22090457, "99.00 percentile latency (ns)": 63948113, "99.90 percentile latency (ns)": 103096118, "Max latency (ns)": 117359647, "Mean latency (ns)": 10254123, "Min duration satisfied": "Yes", "Min latency (ns)": 7831250, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 97.22, "QPS w/o loadgen overhead": 97.52, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 9.939708, "characteristics.90th_percentile_latency_ns": 9939708.0, "characteristics.90th_percentile_latency_s": 0.009939708, "characteristics.90th_percentile_latency_us": 9939.708, "characteristics.accuracy": 61.456, "characteristics.good": 30728, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "12454240936a5f8b", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 29050615, "90.00 percentile latency (ns)": 111500635, "90th percentile latency (ns)": 111500635, "95.00 percentile latency (ns)": 116039769, "97.00 percentile latency (ns)": 118585473, "99.00 percentile latency (ns)": 124206417, "99.90 percentile latency (ns)": 132168918, "Max latency (ns)": 141832409, "Mean latency (ns)": 48846160, "Min duration satisfied": "Yes", "Min latency (ns)": 22974915, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.45, "QPS w/o loadgen overhead": 20.47, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 111.500635, "characteristics.90th_percentile_latency_ns": 111500635.0, "characteristics.90th_percentile_latency_s": 0.111500635, "characteristics.90th_percentile_latency_us": 111500.635, "characteristics.accuracy": 64.348, "characteristics.good": 32174, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.6582, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "9f6e1b85e64c23b3", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10826962, "90.00 percentile latency (ns)": 16691362, "90th percentile latency (ns)": 16691362, "95.00 percentile latency (ns)": 34728431, "97.00 percentile latency (ns)": 75910651, "99.00 percentile latency (ns)": 102874204, "99.90 percentile latency (ns)": 112973233, "Max latency (ns)": 122977561, "Mean latency (ns)": 15088005, "Min duration satisfied": "Yes", "Min latency (ns)": 10338047, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 66.03, "QPS w/o loadgen overhead": 66.28, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 16.691362, "characteristics.90th_percentile_latency_ns": 16691362.0, "characteristics.90th_percentile_latency_s": 0.016691362, "characteristics.90th_percentile_latency_us": 16691.362, "characteristics.accuracy": 62.298, "characteristics.good": 31149, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 62.5, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "47a7841a788eec42", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 31269880, "90.00 percentile latency (ns)": 105890323, "90th percentile latency (ns)": 105890323, "95.00 percentile latency (ns)": 109653984, "97.00 percentile latency (ns)": 111758649, "99.00 percentile latency (ns)": 115922671, "99.90 percentile latency (ns)": 124626965, "Max latency (ns)": 141015695, "Mean latency (ns)": 50104368, "Min duration satisfied": "Yes", "Min latency (ns)": 15566977, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 19.93, "QPS w/o loadgen overhead": 19.96, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 105.890323, "characteristics.90th_percentile_latency_ns": 105890323.0, "characteristics.90th_percentile_latency_s": 0.105890323, "characteristics.90th_percentile_latency_us": 105890.323, "characteristics.accuracy": 63.716, "characteristics.good": 31858, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 17.2414, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "7c8ba5d8a80444a7", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7930698, "90.00 percentile latency (ns)": 9282319, "90th percentile latency (ns)": 9282319, "95.00 percentile latency (ns)": 11159779, "97.00 percentile latency (ns)": 21056606, "99.00 percentile latency (ns)": 63261785, "99.90 percentile latency (ns)": 103318463, "Max latency (ns)": 117994734, "Mean latency (ns)": 9471033, "Min duration satisfied": "Yes", "Min latency (ns)": 7117856, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 105.2, "QPS w/o loadgen overhead": 105.59, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 9.282319, "characteristics.90th_percentile_latency_ns": 9282319.0, "characteristics.90th_percentile_latency_s": 0.009282319, "characteristics.90th_percentile_latency_us": 9282.319, "characteristics.accuracy": 58.46, "characteristics.good": 29230, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 90.9091, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "4c11a707d6fd177f", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 27680589, "90.00 percentile latency (ns)": 111265732, "90th percentile latency (ns)": 111265732, "95.00 percentile latency (ns)": 116376725, "97.00 percentile latency (ns)": 118541431, "99.00 percentile latency (ns)": 122734634, "99.90 percentile latency (ns)": 131910456, "Max latency (ns)": 170061947, "Mean latency (ns)": 47738250, "Min duration satisfied": "Yes", "Min latency (ns)": 23015678, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.92, "QPS w/o loadgen overhead": 20.95, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 111.265732, "characteristics.90th_percentile_latency_ns": 111265732.0, "characteristics.90th_percentile_latency_s": 0.111265732, "characteristics.90th_percentile_latency_us": 111265.732, "characteristics.accuracy": 66.384, "characteristics.good": 33192, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 21.2766, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "f61fd6fd450cbdb2", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11227419, "90.00 percentile latency (ns)": 13519919, "90th percentile latency (ns)": 13519919, "95.00 percentile latency (ns)": 21320382, "97.00 percentile latency (ns)": 32095176, "99.00 percentile latency (ns)": 84510013, "99.90 percentile latency (ns)": 107174909, "Max latency (ns)": 121204556, "Mean latency (ns)": 13308576, "Min duration satisfied": "Yes", "Min latency (ns)": 10308968, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 74.95, "QPS w/o loadgen overhead": 75.14, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 13.519919, "characteristics.90th_percentile_latency_ns": 13519919.0, "characteristics.90th_percentile_latency_s": 0.013519919, "characteristics.90th_percentile_latency_us": 13519.919, "characteristics.accuracy": 64.08, "characteristics.good": 32040, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "4939b2c0621030aa", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 30469794, "90.00 percentile latency (ns)": 104486530, "90th percentile latency (ns)": 104486530, "95.00 percentile latency (ns)": 120078260, "97.00 percentile latency (ns)": 124319548, "99.00 percentile latency (ns)": 131195289, "99.90 percentile latency (ns)": 140987109, "Max latency (ns)": 157011839, "Mean latency (ns)": 45263185, "Min duration satisfied": "Yes", "Min latency (ns)": 29046462, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 22.07, "QPS w/o loadgen overhead": 22.09, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 104.48653, "characteristics.90th_percentile_latency_ns": 104486530.0, "characteristics.90th_percentile_latency_s": 0.10448653, "characteristics.90th_percentile_latency_us": 104486.53, "characteristics.accuracy": 68.242, "characteristics.good": 34121, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.5185, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "4bcc8a8a07d85b09", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 14653339, "90.00 percentile latency (ns)": 18213715, "90th percentile latency (ns)": 18213715, "95.00 percentile latency (ns)": 32555846, "97.00 percentile latency (ns)": 44771434, "99.00 percentile latency (ns)": 99529581, "99.90 percentile latency (ns)": 116789545, "Max latency (ns)": 137581845, "Mean latency (ns)": 17458367, "Min duration satisfied": "Yes", "Min latency (ns)": 13464214, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 57.15, "QPS w/o loadgen overhead": 57.28, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 18.213715, "characteristics.90th_percentile_latency_ns": 18213715.0, "characteristics.90th_percentile_latency_s": 0.018213715, "characteristics.90th_percentile_latency_us": 18213.715, "characteristics.accuracy": 67.24, "characteristics.good": 33620, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 55.5556, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "2ba84f2c895774b0", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 42093860, "90.00 percentile latency (ns)": 103966946, "90th percentile latency (ns)": 103966946, "95.00 percentile latency (ns)": 126031913, "97.00 percentile latency (ns)": 130761700, "99.00 percentile latency (ns)": 137410859, "99.90 percentile latency (ns)": 147584181, "Max latency (ns)": 153754085, "Mean latency (ns)": 55490086, "Min duration satisfied": "Yes", "Min latency (ns)": 39601280, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 18.01, "QPS w/o loadgen overhead": 18.02, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 103.966946, "characteristics.90th_percentile_latency_ns": 103966946.0, "characteristics.90th_percentile_latency_s": 0.103966946, "characteristics.90th_percentile_latency_us": 103966.946, "characteristics.accuracy": 69.262, "characteristics.good": 34631, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 16.129, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "4dac9e263964837a", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 19178790, "90.00 percentile latency (ns)": 38852916, "90th percentile latency (ns)": 38852916, "95.00 percentile latency (ns)": 83031837, "97.00 percentile latency (ns)": 104816710, "99.00 percentile latency (ns)": 113668641, "99.90 percentile latency (ns)": 122058297, "Max latency (ns)": 128536070, "Mean latency (ns)": 26270045, "Min duration satisfied": "Yes", "Min latency (ns)": 18471522, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 37.96, "QPS w/o loadgen overhead": 38.07, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 38.852916, "characteristics.90th_percentile_latency_ns": 38852916.0, "characteristics.90th_percentile_latency_s": 0.038852916, "characteristics.90th_percentile_latency_us": 38852.916, "characteristics.accuracy": 68.126, "characteristics.good": 34063, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 43.4783, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "9d5aafcf3f833428", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 32019478, "90.00 percentile latency (ns)": 112981237, "90th percentile latency (ns)": 112981237, "95.00 percentile latency (ns)": 117429446, "97.00 percentile latency (ns)": 119822574, "99.00 percentile latency (ns)": 124209659, "99.90 percentile latency (ns)": 132818665, "Max latency (ns)": 191395578, "Mean latency (ns)": 52205445, "Min duration satisfied": "Yes", "Min latency (ns)": 23986389, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 19.13, "QPS w/o loadgen overhead": 19.16, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 112.981237, "characteristics.90th_percentile_latency_ns": 112981237.0, "characteristics.90th_percentile_latency_s": 0.112981237, "characteristics.90th_percentile_latency_us": 112981.237, "characteristics.accuracy": 66.604, "characteristics.good": 33302, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.873, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "0bd970248886c37f", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11452855, "90.00 percentile latency (ns)": 13839854, "90th percentile latency (ns)": 13839854, "95.00 percentile latency (ns)": 21852507, "97.00 percentile latency (ns)": 32943390, "99.00 percentile latency (ns)": 82594100, "99.90 percentile latency (ns)": 108376746, "Max latency (ns)": 121321486, "Mean latency (ns)": 13625916, "Min duration satisfied": "Yes", "Min latency (ns)": 10658358, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 73.18, "QPS w/o loadgen overhead": 73.39, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 13.839854, "characteristics.90th_percentile_latency_ns": 13839854.0, "characteristics.90th_percentile_latency_s": 0.013839854, "characteristics.90th_percentile_latency_us": 13839.854, "characteristics.accuracy": 64.768, "characteristics.good": 32384, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "ef4b0bb65cd24101", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 36871071, "90.00 percentile latency (ns)": 109830506, "90th percentile latency (ns)": 109830506, "95.00 percentile latency (ns)": 123202720, "97.00 percentile latency (ns)": 126797048, "99.00 percentile latency (ns)": 133317612, "99.90 percentile latency (ns)": 142310702, "Max latency (ns)": 190740268, "Mean latency (ns)": 52429974, "Min duration satisfied": "Yes", "Min latency (ns)": 34244324, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 19.06, "QPS w/o loadgen overhead": 19.07, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 109.830506, "characteristics.90th_percentile_latency_ns": 109830506.0, "characteristics.90th_percentile_latency_s": 0.109830506, "characteristics.90th_percentile_latency_us": 109830.506, "characteristics.accuracy": 69.106, "characteristics.good": 34553, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 16.9492, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "3683c47e595855c3", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 16706711, "90.00 percentile latency (ns)": 20832059, "90th percentile latency (ns)": 20832059, "95.00 percentile latency (ns)": 34428515, "97.00 percentile latency (ns)": 45665545, "99.00 percentile latency (ns)": 98353047, "99.90 percentile latency (ns)": 116337492, "Max latency (ns)": 134855826, "Mean latency (ns)": 19576610, "Min duration satisfied": "Yes", "Min latency (ns)": 15677421, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 50.99, "QPS w/o loadgen overhead": 51.08, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 20.832059, "characteristics.90th_percentile_latency_ns": 20832059.0, "characteristics.90th_percentile_latency_s": 0.020832059, "characteristics.90th_percentile_latency_us": 20832.059, "characteristics.accuracy": 68.248, "characteristics.good": 34124, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 47.619, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "31c9f37b26e992cc", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 47220940, "90.00 percentile latency (ns)": 99206589, "90th percentile latency (ns)": 99206589, "95.00 percentile latency (ns)": 130194464, "97.00 percentile latency (ns)": 136852304, "99.00 percentile latency (ns)": 146423124, "99.90 percentile latency (ns)": 156444489, "Max latency (ns)": 168851599, "Mean latency (ns)": 59568211, "Min duration satisfied": "Yes", "Min latency (ns)": 44499111, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.78, "QPS w/o loadgen overhead": 16.79, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 99.206589, "characteristics.90th_percentile_latency_ns": 99206589.0, "characteristics.90th_percentile_latency_s": 0.099206589, "characteristics.90th_percentile_latency_us": 99206.589, "characteristics.accuracy": 70.626, "characteristics.good": 35313, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.625, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "4c6b9267f0840289", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 22883863, "90.00 percentile latency (ns)": 31358527, "90th percentile latency (ns)": 31358527, "95.00 percentile latency (ns)": 46005227, "97.00 percentile latency (ns)": 62535262, "99.00 percentile latency (ns)": 112243495, "99.90 percentile latency (ns)": 125623398, "Max latency (ns)": 135226530, "Mean latency (ns)": 26546330, "Min duration satisfied": "Yes", "Min latency (ns)": 21253238, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 37.61, "QPS w/o loadgen overhead": 37.67, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 31.358527, "characteristics.90th_percentile_latency_ns": 31358527.0, "characteristics.90th_percentile_latency_s": 0.031358527, "characteristics.90th_percentile_latency_us": 31358.527, "characteristics.accuracy": 70.04, "characteristics.good": 35020, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 37.037, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "2e86e21427db31b6", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 67113534, "90.00 percentile latency (ns)": 112243387, "90th percentile latency (ns)": 112243387, "95.00 percentile latency (ns)": 140090271, "97.00 percentile latency (ns)": 147007330, "99.00 percentile latency (ns)": 154845317, "99.90 percentile latency (ns)": 164764637, "Max latency (ns)": 173258843, "Mean latency (ns)": 75139987, "Min duration satisfied": "Yes", "Min latency (ns)": 58561334, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 13.3, "QPS w/o loadgen overhead": 13.31, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 112.243387, "characteristics.90th_percentile_latency_ns": 112243387.0, "characteristics.90th_percentile_latency_s": 0.112243387, "characteristics.90th_percentile_latency_us": 112243.387, "characteristics.accuracy": 71.676, "characteristics.good": 35838, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.987, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "77a87c377334797b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 29471510, "90.00 percentile latency (ns)": 59180448, "90th percentile latency (ns)": 59180448, "95.00 percentile latency (ns)": 111633316, "97.00 percentile latency (ns)": 120648137, "99.00 percentile latency (ns)": 127563838, "99.90 percentile latency (ns)": 138155778, "Max latency (ns)": 144373010, "Mean latency (ns)": 38957607, "Min duration satisfied": "Yes", "Min latency (ns)": 28517759, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 25.62, "QPS w/o loadgen overhead": 25.67, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 59.180448, "characteristics.90th_percentile_latency_ns": 59180448.0, "characteristics.90th_percentile_latency_s": 0.059180448, "characteristics.90th_percentile_latency_us": 59180.448, "characteristics.accuracy": 70.772, "characteristics.good": 35386, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 24.3902, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "2c45effb1643c1b9", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10213005, "90.00 percentile latency (ns)": 71072321, "90th percentile latency (ns)": 71072321, "95.00 percentile latency (ns)": 122850958, "97.00 percentile latency (ns)": 149072712, "99.00 percentile latency (ns)": 165629398, "99.90 percentile latency (ns)": 178656305, "Max latency (ns)": 204399481, "Mean latency (ns)": 24784573, "Min duration satisfied": "Yes", "Min latency (ns)": 8405253, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 40.28, "QPS w/o loadgen overhead": 40.35, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 71.072321, "characteristics.90th_percentile_latency_ns": 71072321.0, "characteristics.90th_percentile_latency_s": 0.071072321, "characteristics.90th_percentile_latency_us": 71072.321, "characteristics.accuracy": 52.172, "characteristics.good": 26086, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 38.4615, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "0fbfabcfede81bba", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 5205994, "90.00 percentile latency (ns)": 5548734, "90th percentile latency (ns)": 5548734, "95.00 percentile latency (ns)": 6875236, "97.00 percentile latency (ns)": 7661844, "99.00 percentile latency (ns)": 27438555, "99.90 percentile latency (ns)": 139583853, "Max latency (ns)": 189544897, "Mean latency (ns)": 5910820, "Min duration satisfied": "Yes", "Min latency (ns)": 4503670, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 168.24, "QPS w/o loadgen overhead": 169.18, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 5.548734, "characteristics.90th_percentile_latency_ns": 5548734.0, "characteristics.90th_percentile_latency_s": 0.005548734, "characteristics.90th_percentile_latency_us": 5548.734, "characteristics.accuracy": 49.12, "characteristics.good": 24560, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "b84cef39c374b778", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 18858132, "90.00 percentile latency (ns)": 146217739, "90th percentile latency (ns)": 146217739, "95.00 percentile latency (ns)": 163071607, "97.00 percentile latency (ns)": 169737105, "99.00 percentile latency (ns)": 178508725, "99.90 percentile latency (ns)": 191660267, "Max latency (ns)": 239562484, "Mean latency (ns)": 46976665, "Min duration satisfied": "Yes", "Min latency (ns)": 11400664, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 21.26, "QPS w/o loadgen overhead": 21.29, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 146.217739, "characteristics.90th_percentile_latency_ns": 146217739.0, "characteristics.90th_percentile_latency_s": 0.146217739, "characteristics.90th_percentile_latency_us": 146217.739, "characteristics.accuracy": 56.988, "characteristics.good": 28494, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.5185, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "843fc897a061eb57", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7051358, "90.00 percentile latency (ns)": 7642861, "90th percentile latency (ns)": 7642861, "95.00 percentile latency (ns)": 9089813, "97.00 percentile latency (ns)": 11290455, "99.00 percentile latency (ns)": 36475044, "99.90 percentile latency (ns)": 159079489, "Max latency (ns)": 183746471, "Mean latency (ns)": 8050293, "Min duration satisfied": "Yes", "Min latency (ns)": 6052109, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 123.73, "QPS w/o loadgen overhead": 124.22, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 7.642861, "characteristics.90th_percentile_latency_ns": 7642861.0, "characteristics.90th_percentile_latency_s": 0.007642861, "characteristics.90th_percentile_latency_us": 7642.861, "characteristics.accuracy": 54.568, "characteristics.good": 27284, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "ba16461a889aea42", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 21515470, "90.00 percentile latency (ns)": 148947822, "90th percentile latency (ns)": 148947822, "95.00 percentile latency (ns)": 168945597, "97.00 percentile latency (ns)": 174662026, "99.00 percentile latency (ns)": 181346891, "99.90 percentile latency (ns)": 192841867, "Max latency (ns)": 203763581, "Mean latency (ns)": 49312902, "Min duration satisfied": "Yes", "Min latency (ns)": 14944785, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.26, "QPS w/o loadgen overhead": 20.28, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 148.947822, "characteristics.90th_percentile_latency_ns": 148947822.0, "characteristics.90th_percentile_latency_s": 0.148947822, "characteristics.90th_percentile_latency_us": 148947.822, "characteristics.accuracy": 59.322, "characteristics.good": 29661, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.625, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "89b227e8ee12ec42", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9210561, "90.00 percentile latency (ns)": 10080606, "90th percentile latency (ns)": 10080606, "95.00 percentile latency (ns)": 12425320, "97.00 percentile latency (ns)": 22780588, "99.00 percentile latency (ns)": 65421873, "99.90 percentile latency (ns)": 174927510, "Max latency (ns)": 205106634, "Mean latency (ns)": 10804334, "Min duration satisfied": "Yes", "Min latency (ns)": 7813474, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 92.25, "QPS w/o loadgen overhead": 92.56, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 10.080606, "characteristics.90th_percentile_latency_ns": 10080606.0, "characteristics.90th_percentile_latency_s": 0.010080606, "characteristics.90th_percentile_latency_us": 10080.606, "characteristics.accuracy": 56.594, "characteristics.good": 28297, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "6e3914d04599b3b0", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 21261351, "90.00 percentile latency (ns)": 118634059, "90th percentile latency (ns)": 118634059, "95.00 percentile latency (ns)": 172336421, "97.00 percentile latency (ns)": 180496403, "99.00 percentile latency (ns)": 188827033, "99.90 percentile latency (ns)": 199542401, "Max latency (ns)": 209096039, "Mean latency (ns)": 42982582, "Min duration satisfied": "Yes", "Min latency (ns)": 19823441, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 23.25, "QPS w/o loadgen overhead": 23.27, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 118.634059, "characteristics.90th_percentile_latency_ns": 118634059.0, "characteristics.90th_percentile_latency_s": 0.118634059, "characteristics.90th_percentile_latency_us": 118634.059, "characteristics.accuracy": 60.976, "characteristics.good": 30488, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.873, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "dd78a44019078a83", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10464424, "90.00 percentile latency (ns)": 13550840, "90th percentile latency (ns)": 13550840, "95.00 percentile latency (ns)": 24924679, "97.00 percentile latency (ns)": 39612849, "99.00 percentile latency (ns)": 144004531, "99.90 percentile latency (ns)": 182001982, "Max latency (ns)": 197312885, "Mean latency (ns)": 14159493, "Min duration satisfied": "Yes", "Min latency (ns)": 10007963, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 70.41, "QPS w/o loadgen overhead": 70.62, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 13.55084, "characteristics.90th_percentile_latency_ns": 13550840.0, "characteristics.90th_percentile_latency_s": 0.01355084, "characteristics.90th_percentile_latency_us": 13550.84, "characteristics.accuracy": 58.528, "characteristics.good": 29264, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 62.5, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "483ebc62f8c3db8a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 10114420, "90.00 percentile latency (ns)": 74626215, "90th percentile latency (ns)": 74626215, "95.00 percentile latency (ns)": 100134153, "97.00 percentile latency (ns)": 119094772, "99.00 percentile latency (ns)": 140332201, "99.90 percentile latency (ns)": 159598885, "Max latency (ns)": 174876717, "Mean latency (ns)": 25676260, "Min duration satisfied": "Yes", "Min latency (ns)": 6165537, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 38.86, "QPS w/o loadgen overhead": 38.95, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 74.626215, "characteristics.90th_percentile_latency_ns": 74626215.0, "characteristics.90th_percentile_latency_s": 0.074626215, "characteristics.90th_percentile_latency_us": 74626.215, "characteristics.accuracy": 47.14, "characteristics.good": 23570, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.35_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "c8e7b2a4978afa53", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3665669, "90.00 percentile latency (ns)": 3910669, "90th percentile latency (ns)": 3910669, "95.00 percentile latency (ns)": 4677169, "97.00 percentile latency (ns)": 5617795, "99.00 percentile latency (ns)": 18406510, "99.90 percentile latency (ns)": 106049478, "Max latency (ns)": 168409023, "Mean latency (ns)": 4138556, "Min duration satisfied": "Yes", "Min latency (ns)": 3153794, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 239.27, "QPS w/o loadgen overhead": 241.63, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 3.910669, "characteristics.90th_percentile_latency_ns": 3910669.0, "characteristics.90th_percentile_latency_s": 0.003910669, "characteristics.90th_percentile_latency_us": 3910.669, "characteristics.accuracy": 43.468, "characteristics.good": 21734, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "668085840647b4f4", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 19576646, "90.00 percentile latency (ns)": 152375822, "90th percentile latency (ns)": 152375822, "95.00 percentile latency (ns)": 168325695, "97.00 percentile latency (ns)": 173458124, "99.00 percentile latency (ns)": 180401950, "99.90 percentile latency (ns)": 193479425, "Max latency (ns)": 207864628, "Mean latency (ns)": 50274458, "Min duration satisfied": "Yes", "Min latency (ns)": 10450992, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 19.87, "QPS w/o loadgen overhead": 19.89, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 152.375822, "characteristics.90th_percentile_latency_ns": 152375822.0, "characteristics.90th_percentile_latency_s": 0.152375822, "characteristics.90th_percentile_latency_us": 152375.822, "characteristics.accuracy": 58.402, "characteristics.good": 29201, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 16.3934, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "41a4497bb8446d4e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6553749, "90.00 percentile latency (ns)": 7134169, "90th percentile latency (ns)": 7134169, "95.00 percentile latency (ns)": 8682334, "97.00 percentile latency (ns)": 10252375, "99.00 percentile latency (ns)": 34875174, "99.90 percentile latency (ns)": 164025206, "Max latency (ns)": 192250966, "Mean latency (ns)": 7629007, "Min duration satisfied": "Yes", "Min latency (ns)": 5639085, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 130.49, "QPS w/o loadgen overhead": 131.08, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 7.134169, "characteristics.90th_percentile_latency_ns": 7134169.0, "characteristics.90th_percentile_latency_s": 0.007134169, "characteristics.90th_percentile_latency_us": 7134.169, "characteristics.accuracy": 55.54, "characteristics.good": 27770, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "e7344c884fdec257", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20302919, "90.00 percentile latency (ns)": 149681596, "90th percentile latency (ns)": 149681596, "95.00 percentile latency (ns)": 171478140, "97.00 percentile latency (ns)": 177026518, "99.00 percentile latency (ns)": 185637602, "99.90 percentile latency (ns)": 199486606, "Max latency (ns)": 209828482, "Mean latency (ns)": 48166814, "Min duration satisfied": "Yes", "Min latency (ns)": 14214377, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.74, "QPS w/o loadgen overhead": 20.76, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 149.681596, "characteristics.90th_percentile_latency_ns": 149681596.0, "characteristics.90th_percentile_latency_s": 0.149681596, "characteristics.90th_percentile_latency_us": 149681.596, "characteristics.accuracy": 61.668, "characteristics.good": 30834, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.9254, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "0496adc35616d436", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8706610, "90.00 percentile latency (ns)": 9571626, "90th percentile latency (ns)": 9571626, "95.00 percentile latency (ns)": 11733853, "97.00 percentile latency (ns)": 20562377, "99.00 percentile latency (ns)": 50446793, "99.90 percentile latency (ns)": 172257479, "Max latency (ns)": 196912385, "Mean latency (ns)": 10222979, "Min duration satisfied": "Yes", "Min latency (ns)": 7546065, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 97.51, "QPS w/o loadgen overhead": 97.82, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 9.571626, "characteristics.90th_percentile_latency_ns": 9571626.0, "characteristics.90th_percentile_latency_s": 0.009571626, "characteristics.90th_percentile_latency_us": 9571.626, "characteristics.accuracy": 59.226, "characteristics.good": 29613, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "72f84d72e844f1d8", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20220711, "90.00 percentile latency (ns)": 88496231, "90th percentile latency (ns)": 88496231, "95.00 percentile latency (ns)": 159634421, "97.00 percentile latency (ns)": 174246441, "99.00 percentile latency (ns)": 184792827, "99.90 percentile latency (ns)": 197817499, "Max latency (ns)": 217539693, "Mean latency (ns)": 37623406, "Min duration satisfied": "Yes", "Min latency (ns)": 18862706, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 26.55, "QPS w/o loadgen overhead": 26.58, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 88.496231, "characteristics.90th_percentile_latency_ns": 88496231.0, "characteristics.90th_percentile_latency_s": 0.088496231, "characteristics.90th_percentile_latency_us": 88496.231, "characteristics.accuracy": 64.282, "characteristics.good": 32141, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 25.641, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "ec4417a4c2df04e9", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11609135, "90.00 percentile latency (ns)": 13368499, "90th percentile latency (ns)": 13368499, "95.00 percentile latency (ns)": 22314571, "97.00 percentile latency (ns)": 33863360, "99.00 percentile latency (ns)": 97990265, "99.90 percentile latency (ns)": 182462430, "Max latency (ns)": 202069877, "Mean latency (ns)": 13921083, "Min duration satisfied": "Yes", "Min latency (ns)": 10051966, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 71.65, "QPS w/o loadgen overhead": 71.83, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 13.368499, "characteristics.90th_percentile_latency_ns": 13368499.0, "characteristics.90th_percentile_latency_s": 0.013368499, "characteristics.90th_percentile_latency_us": 13368.499, "characteristics.accuracy": 61.924, "characteristics.good": 30962, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "eb4210815876e263", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 36431825, "90.00 percentile latency (ns)": 175026999, "90th percentile latency (ns)": 175026999, "95.00 percentile latency (ns)": 187267887, "97.00 percentile latency (ns)": 191344378, "99.00 percentile latency (ns)": 197712879, "99.90 percentile latency (ns)": 210563221, "Max latency (ns)": 223541360, "Mean latency (ns)": 65653870, "Min duration satisfied": "Yes", "Min latency (ns)": 25452903, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 15.22, "QPS w/o loadgen overhead": 15.23, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 175.026999, "characteristics.90th_percentile_latency_ns": 175026999.0, "characteristics.90th_percentile_latency_s": 0.175026999, "characteristics.90th_percentile_latency_us": 175026.999, "characteristics.accuracy": 65.75, "characteristics.good": 32875, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.2857, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "909fc64ba0a04670", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 13528391, "90.00 percentile latency (ns)": 19582815, "90th percentile latency (ns)": 19582815, "95.00 percentile latency (ns)": 35533208, "97.00 percentile latency (ns)": 54296231, "99.00 percentile latency (ns)": 149126253, "99.90 percentile latency (ns)": 185098806, "Max latency (ns)": 202497228, "Mean latency (ns)": 18015005, "Min duration satisfied": "Yes", "Min latency (ns)": 13010682, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 55.37, "QPS w/o loadgen overhead": 55.51, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 19.582815, "characteristics.90th_percentile_latency_ns": 19582815.0, "characteristics.90th_percentile_latency_s": 0.019582815, "characteristics.90th_percentile_latency_us": 19582.815, "characteristics.accuracy": 63.542, "characteristics.good": 31771, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 38.4615, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "d4c6bbd060542199", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 12780032, "90.00 percentile latency (ns)": 87989630, "90th percentile latency (ns)": 87989630, "95.00 percentile latency (ns)": 128423714, "97.00 percentile latency (ns)": 146326964, "99.00 percentile latency (ns)": 161816515, "99.90 percentile latency (ns)": 175251622, "Max latency (ns)": 221374888, "Mean latency (ns)": 32478406, "Min duration satisfied": "Yes", "Min latency (ns)": 7508101, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 30.74, "QPS w/o loadgen overhead": 30.79, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 87.98963, "characteristics.90th_percentile_latency_ns": 87989630.0, "characteristics.90th_percentile_latency_s": 0.08798963, "characteristics.90th_percentile_latency_us": 87989.63, "characteristics.accuracy": 52.446, "characteristics.good": 26223, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.5_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.2558, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "a78e9bf4584a7f8b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4536301, "90.00 percentile latency (ns)": 4827087, "90th percentile latency (ns)": 4827087, "95.00 percentile latency (ns)": 5834796, "97.00 percentile latency (ns)": 7165379, "99.00 percentile latency (ns)": 25497262, "99.90 percentile latency (ns)": 129616094, "Max latency (ns)": 177859017, "Mean latency (ns)": 5248003, "Min duration satisfied": "Yes", "Min latency (ns)": 4066717, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 189.23, "QPS w/o loadgen overhead": 190.55, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 4.827087, "characteristics.90th_percentile_latency_ns": 4827087.0, "characteristics.90th_percentile_latency_s": 0.004827087, "characteristics.90th_percentile_latency_us": 4827.087, "characteristics.accuracy": 49.11, "characteristics.good": 24555, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "4304ae8c179b49c6", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 28009350, "90.00 percentile latency (ns)": 163724531, "90th percentile latency (ns)": 163724531, "95.00 percentile latency (ns)": 176483828, "97.00 percentile latency (ns)": 181654451, "99.00 percentile latency (ns)": 188445082, "99.90 percentile latency (ns)": 197133799, "Max latency (ns)": 204222224, "Mean latency (ns)": 58102036, "Min duration satisfied": "Yes", "Min latency (ns)": 16093297, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.2, "QPS w/o loadgen overhead": 17.21, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 163.724531, "characteristics.90th_percentile_latency_ns": 163724531.0, "characteristics.90th_percentile_latency_s": 0.163724531, "characteristics.90th_percentile_latency_us": 163724.531, "characteristics.accuracy": 64.038, "characteristics.good": 32019, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.5, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "00e7eaa94c74400a", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9345883, "90.00 percentile latency (ns)": 10262592, "90th percentile latency (ns)": 10262592, "95.00 percentile latency (ns)": 13389261, "97.00 percentile latency (ns)": 23671271, "99.00 percentile latency (ns)": 52784919, "99.90 percentile latency (ns)": 169222984, "Max latency (ns)": 190353819, "Mean latency (ns)": 11014115, "Min duration satisfied": "Yes", "Min latency (ns)": 8397382, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 90.52, "QPS w/o loadgen overhead": 90.79, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 10.262592, "characteristics.90th_percentile_latency_ns": 10262592.0, "characteristics.90th_percentile_latency_s": 0.010262592, "characteristics.90th_percentile_latency_us": 10262.592, "characteristics.accuracy": 62.07, "characteristics.good": 31035, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "8a633574d5ca45a9", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 28104014, "90.00 percentile latency (ns)": 167782405, "90th percentile latency (ns)": 167782405, "95.00 percentile latency (ns)": 184891050, "97.00 percentile latency (ns)": 189879478, "99.00 percentile latency (ns)": 196898045, "99.90 percentile latency (ns)": 207462820, "Max latency (ns)": 224065580, "Mean latency (ns)": 57069109, "Min duration satisfied": "Yes", "Min latency (ns)": 22335162, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.51, "QPS w/o loadgen overhead": 17.52, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 167.782405, "characteristics.90th_percentile_latency_ns": 167782405.0, "characteristics.90th_percentile_latency_s": 0.167782405, "characteristics.90th_percentile_latency_us": 167782.405, "characteristics.accuracy": 66.75, "characteristics.good": 33375, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 21.2766, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "8d02265fb402596d", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 13515873, "90.00 percentile latency (ns)": 15632213, "90th percentile latency (ns)": 15632213, "95.00 percentile latency (ns)": 26927257, "97.00 percentile latency (ns)": 38393193, "99.00 percentile latency (ns)": 78908592, "99.90 percentile latency (ns)": 182317462, "Max latency (ns)": 200299919, "Mean latency (ns)": 15952584, "Min duration satisfied": "Yes", "Min latency (ns)": 12071554, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 62.56, "QPS w/o loadgen overhead": 62.69, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 15.632213, "characteristics.90th_percentile_latency_ns": 15632213.0, "characteristics.90th_percentile_latency_s": 0.015632213, "characteristics.90th_percentile_latency_us": 15632.213, "characteristics.accuracy": 64.972, "characteristics.good": 32486, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 41.6667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "ee115d787494d409", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 30922372, "90.00 percentile latency (ns)": 100687698, "90th percentile latency (ns)": 100687698, "95.00 percentile latency (ns)": 166090576, "97.00 percentile latency (ns)": 183803355, "99.00 percentile latency (ns)": 195721928, "99.90 percentile latency (ns)": 208600006, "Max latency (ns)": 222734699, "Mean latency (ns)": 49149669, "Min duration satisfied": "Yes", "Min latency (ns)": 29720316, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.33, "QPS w/o loadgen overhead": 20.35, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 100.687698, "characteristics.90th_percentile_latency_ns": 100687698.0, "characteristics.90th_percentile_latency_s": 0.100687698, "characteristics.90th_percentile_latency_us": 100687.698, "characteristics.accuracy": 68.834, "characteristics.good": 34417, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 13.8889, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "8f4115997870b5b8", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 17615222, "90.00 percentile latency (ns)": 22136642, "90th percentile latency (ns)": 22136642, "95.00 percentile latency (ns)": 38915758, "97.00 percentile latency (ns)": 56554938, "99.00 percentile latency (ns)": 152838474, "99.90 percentile latency (ns)": 195292286, "Max latency (ns)": 207265853, "Mean latency (ns)": 21510970, "Min duration satisfied": "Yes", "Min latency (ns)": 15719395, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 46.39, "QPS w/o loadgen overhead": 46.49, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 22.136642, "characteristics.90th_percentile_latency_ns": 22136642.0, "characteristics.90th_percentile_latency_s": 0.022136642, "characteristics.90th_percentile_latency_us": 22136.642, "characteristics.accuracy": 67.474, "characteristics.good": 33737, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "2b851515b1cf6976", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 44642272, "90.00 percentile latency (ns)": 147897162, "90th percentile latency (ns)": 147897162, "95.00 percentile latency (ns)": 192843801, "97.00 percentile latency (ns)": 201030289, "99.00 percentile latency (ns)": 209919503, "99.90 percentile latency (ns)": 221491487, "Max latency (ns)": 229554586, "Mean latency (ns)": 66022015, "Min duration satisfied": "Yes", "Min latency (ns)": 40152632, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 15.14, "QPS w/o loadgen overhead": 15.15, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 147.897162, "characteristics.90th_percentile_latency_ns": 147897162.0, "characteristics.90th_percentile_latency_s": 0.147897162, "characteristics.90th_percentile_latency_us": 147897.162, "characteristics.accuracy": 70.122, "characteristics.good": 35061, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.7059, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "89813ee909403d93", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 21574324, "90.00 percentile latency (ns)": 36915725, "90th percentile latency (ns)": 36915725, "95.00 percentile latency (ns)": 57276216, "97.00 percentile latency (ns)": 108585630, "99.00 percentile latency (ns)": 179276399, "99.90 percentile latency (ns)": 198989599, "Max latency (ns)": 215087503, "Mean latency (ns)": 28822997, "Min duration satisfied": "Yes", "Min latency (ns)": 20932366, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 34.64, "QPS w/o loadgen overhead": 34.69, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 36.915725, "characteristics.90th_percentile_latency_ns": 36915725.0, "characteristics.90th_percentile_latency_s": 0.036915725, "characteristics.90th_percentile_latency_us": 36915.725, "characteristics.accuracy": 68.734, "characteristics.good": 34367, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 27.7778, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "43ac94af54c97977", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 19473532, "90.00 percentile latency (ns)": 144222804, "90th percentile latency (ns)": 144222804, "95.00 percentile latency (ns)": 165272168, "97.00 percentile latency (ns)": 170697082, "99.00 percentile latency (ns)": 178262160, "99.90 percentile latency (ns)": 191207190, "Max latency (ns)": 220197485, "Mean latency (ns)": 46844565, "Min duration satisfied": "Yes", "Min latency (ns)": 11443613, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 21.33, "QPS w/o loadgen overhead": 21.35, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 144.222804, "characteristics.90th_percentile_latency_ns": 144222804.0, "characteristics.90th_percentile_latency_s": 0.144222804, "characteristics.90th_percentile_latency_us": 144222.804, "characteristics.accuracy": 59.822, "characteristics.good": 29911, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_0.75_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.7059, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "3ab5d202aff34f33", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6363308, "90.00 percentile latency (ns)": 6912225, "90th percentile latency (ns)": 6912225, "95.00 percentile latency (ns)": 8768984, "97.00 percentile latency (ns)": 12195198, "99.00 percentile latency (ns)": 35451960, "99.90 percentile latency (ns)": 160676373, "Max latency (ns)": 190830690, "Mean latency (ns)": 7541181, "Min duration satisfied": "Yes", "Min latency (ns)": 5678180, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 131.99, "QPS w/o loadgen overhead": 132.61, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 6.912225, "characteristics.90th_percentile_latency_ns": 6912225.0, "characteristics.90th_percentile_latency_s": 0.006912225, "characteristics.90th_percentile_latency_us": 6912.225, "characteristics.accuracy": 57.002, "characteristics.good": 28501, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "7881a5c9df4afeb2", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 26512340, "90.00 percentile latency (ns)": 169755088, "90th percentile latency (ns)": 169755088, "95.00 percentile latency (ns)": 183360654, "97.00 percentile latency (ns)": 188454435, "99.00 percentile latency (ns)": 195369950, "99.90 percentile latency (ns)": 205849329, "Max latency (ns)": 216695538, "Mean latency (ns)": 58189980, "Min duration satisfied": "Yes", "Min latency (ns)": 19906298, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.17, "QPS w/o loadgen overhead": 17.19, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 169.755088, "characteristics.90th_percentile_latency_ns": 169755088.0, "characteristics.90th_percentile_latency_s": 0.169755088, "characteristics.90th_percentile_latency_us": 169755.088, "characteristics.accuracy": 66.036, "characteristics.good": 33018, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-128-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-128-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_128.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.1515, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "2dba847158b7cf3e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11149264, "90.00 percentile latency (ns)": 13011557, "90th percentile latency (ns)": 13011557, "95.00 percentile latency (ns)": 21248080, "97.00 percentile latency (ns)": 30889896, "99.00 percentile latency (ns)": 76446098, "99.90 percentile latency (ns)": 178062297, "Max latency (ns)": 198543587, "Mean latency (ns)": 13432322, "Min duration satisfied": "Yes", "Min latency (ns)": 10083519, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 74.25, "QPS w/o loadgen overhead": 74.45, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 13.011557, "characteristics.90th_percentile_latency_ns": 13011557.0, "characteristics.90th_percentile_latency_s": 0.013011557, "characteristics.90th_percentile_latency_us": 13011.557, "characteristics.accuracy": 63.454, "characteristics.good": 31727, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "4962838d5772638b", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 29477284, "90.00 percentile latency (ns)": 112832957, "90th percentile latency (ns)": 112832957, "95.00 percentile latency (ns)": 179696685, "97.00 percentile latency (ns)": 192010920, "99.00 percentile latency (ns)": 202799284, "99.90 percentile latency (ns)": 211760180, "Max latency (ns)": 218109470, "Mean latency (ns)": 49641921, "Min duration satisfied": "Yes", "Min latency (ns)": 28520618, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 20.13, "QPS w/o loadgen overhead": 20.14, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 112.832957, "characteristics.90th_percentile_latency_ns": 112832957.0, "characteristics.90th_percentile_latency_s": 0.112832957, "characteristics.90th_percentile_latency_us": 112832.957, "characteristics.accuracy": 69.016, "characteristics.good": 34508, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-160-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-160-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_160.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 20.4082, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "fdd44ba973ea79f1", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 16382042, "90.00 percentile latency (ns)": 20234668, "90th percentile latency (ns)": 20234668, "95.00 percentile latency (ns)": 35495210, "97.00 percentile latency (ns)": 50017293, "99.00 percentile latency (ns)": 129623381, "99.90 percentile latency (ns)": 191492889, "Max latency (ns)": 209920090, "Mean latency (ns)": 19882047, "Min duration satisfied": "Yes", "Min latency (ns)": 14825709, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 50.19, "QPS w/o loadgen overhead": 50.3, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 20.234668, "characteristics.90th_percentile_latency_ns": 20234668.0, "characteristics.90th_percentile_latency_s": 0.020234668, "characteristics.90th_percentile_latency_us": 20234.668, "characteristics.accuracy": 67.82, "characteristics.good": 33910, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 50, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "5cdae1f4bd234a3a", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 40567550, "90.00 percentile latency (ns)": 136545072, "90th percentile latency (ns)": 136545072, "95.00 percentile latency (ns)": 193013373, "97.00 percentile latency (ns)": 202012504, "99.00 percentile latency (ns)": 210668304, "99.90 percentile latency (ns)": 220430705, "Max latency (ns)": 227976863, "Mean latency (ns)": 61607395, "Min duration satisfied": "Yes", "Min latency (ns)": 37231459, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.23, "QPS w/o loadgen overhead": 16.23, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 136.545072, "characteristics.90th_percentile_latency_ns": 136545072.0, "characteristics.90th_percentile_latency_s": 0.136545072, "characteristics.90th_percentile_latency_us": 136545.072, "characteristics.accuracy": 70.738, "characteristics.good": 35369, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-192-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-192-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_192.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.2857, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "347d1b0454696f28", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 21975023, "90.00 percentile latency (ns)": 29024868, "90th percentile latency (ns)": 29024868, "95.00 percentile latency (ns)": 44865639, "97.00 percentile latency (ns)": 61585670, "99.00 percentile latency (ns)": 148401649, "99.90 percentile latency (ns)": 196581733, "Max latency (ns)": 219009179, "Mean latency (ns)": 25987340, "Min duration satisfied": "Yes", "Min latency (ns)": 19767113, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 38.42, "QPS w/o loadgen overhead": 38.48, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 29.024868, "characteristics.90th_percentile_latency_ns": 29024868.0, "characteristics.90th_percentile_latency_s": 0.029024868, "characteristics.90th_percentile_latency_us": 29024.868, "characteristics.accuracy": 69.604, "characteristics.good": 34802, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 35.7143, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "2f4f42848725c0de", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 55579410, "90.00 percentile latency (ns)": 130705422, "90th percentile latency (ns)": 130705422, "95.00 percentile latency (ns)": 194662733, "97.00 percentile latency (ns)": 209757384, "99.00 percentile latency (ns)": 221322568, "99.90 percentile latency (ns)": 231422724, "Max latency (ns)": 241701021, "Mean latency (ns)": 72978069, "Min duration satisfied": "Yes", "Min latency (ns)": 50042207, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 13.7, "QPS w/o loadgen overhead": 13.7, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 130.705422, "characteristics.90th_percentile_latency_ns": 130705422.0, "characteristics.90th_percentile_latency_s": 0.130705422, "characteristics.90th_percentile_latency_us": 130705.422, "characteristics.accuracy": 71.77, "characteristics.good": 35885, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 11.4943, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "f671514bd084e601", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 26494628, "90.00 percentile latency (ns)": 49034181, "90th percentile latency (ns)": 49034181, "95.00 percentile latency (ns)": 92485159, "97.00 percentile latency (ns)": 155239706, "99.00 percentile latency (ns)": 194591374, "99.90 percentile latency (ns)": 208654179, "Max latency (ns)": 222012130, "Mean latency (ns)": 36554990, "Min duration satisfied": "Yes", "Min latency (ns)": 25653508, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 27.32, "QPS w/o loadgen overhead": 27.36, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 49.034181, "characteristics.90th_percentile_latency_ns": 49034181.0, "characteristics.90th_percentile_latency_s": 0.049034181, "characteristics.90th_percentile_latency_us": 49034.181, "characteristics.accuracy": 70.542, "characteristics.good": 35271, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 25.641, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "de3f4a83e28744a3", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 22491892, "90.00 percentile latency (ns)": 159527287, "90th percentile latency (ns)": 159527287, "95.00 percentile latency (ns)": 175158210, "97.00 percentile latency (ns)": 180113523, "99.00 percentile latency (ns)": 186679424, "99.90 percentile latency (ns)": 196358300, "Max latency (ns)": 207891714, "Mean latency (ns)": 52277473, "Min duration satisfied": "Yes", "Min latency (ns)": 14357010, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 19.11, "QPS w/o loadgen overhead": 19.13, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 159.527287, "characteristics.90th_percentile_latency_ns": 159527287.0, "characteristics.90th_percentile_latency_s": 0.159527287, "characteristics.90th_percentile_latency_us": 159527.287, "characteristics.accuracy": 61.224, "characteristics.good": 30612, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-96-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-96-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.0_96.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.0845, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "1a7372e224b54848", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 7517471, "90.00 percentile latency (ns)": 8377548, "90th percentile latency (ns)": 8377548, "95.00 percentile latency (ns)": 10177424, "97.00 percentile latency (ns)": 16899539, "99.00 percentile latency (ns)": 43189895, "99.90 percentile latency (ns)": 164456911, "Max latency (ns)": 194712795, "Mean latency (ns)": 8889307, "Min duration satisfied": "Yes", "Min latency (ns)": 6826489, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 112.08, "QPS w/o loadgen overhead": 112.49, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 8.377548, "characteristics.90th_percentile_latency_ns": 8377548.0, "characteristics.90th_percentile_latency_s": 0.008377548, "characteristics.90th_percentile_latency_us": 8377.548, "characteristics.accuracy": 59.12, "characteristics.good": 29560, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 100, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "ad47a788c4088391", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 90824085, "90.00 percentile latency (ns)": 214439020, "90th percentile latency (ns)": 214439020, "95.00 percentile latency (ns)": 231803674, "97.00 percentile latency (ns)": 237869912, "99.00 percentile latency (ns)": 246492457, "99.90 percentile latency (ns)": 258797746, "Max latency (ns)": 265246279, "Mean latency (ns)": 115129795, "Min duration satisfied": "Yes", "Min latency (ns)": 75316119, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 8.68, "QPS w/o loadgen overhead": 8.69, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 214.43902, "characteristics.90th_percentile_latency_ns": 214439020.0, "characteristics.90th_percentile_latency_s": 0.21443902, "characteristics.90th_percentile_latency_us": 214439.02, "characteristics.accuracy": 75.012, "characteristics.good": 37506, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.3-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.3-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.3_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 7.46269, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "06a4dc3848124125", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 97122266, "90.00 percentile latency (ns)": 224166652, "90th percentile latency (ns)": 224166652, "95.00 percentile latency (ns)": 239890229, "97.00 percentile latency (ns)": 245796986, "99.00 percentile latency (ns)": 254903706, "99.90 percentile latency (ns)": 269859817, "Max latency (ns)": 302025508, "Mean latency (ns)": 122297193, "Min duration satisfied": "Yes", "Min latency (ns)": 80872906, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 8.17, "QPS w/o loadgen overhead": 8.18, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 224.166652, "characteristics.90th_percentile_latency_ns": 224166652.0, "characteristics.90th_percentile_latency_s": 0.224166652, "characteristics.90th_percentile_latency_us": 224166.652, "characteristics.accuracy": 75.69, "characteristics.good": 37845, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.4-224-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.4-224-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2266646/files/mobilenet_v2_1.4_224.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 7.8125, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "ea8abda40db1493b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 40968398, "90.00 percentile latency (ns)": 69597610, "90th percentile latency (ns)": 69597610, "95.00 percentile latency (ns)": 123156810, "97.00 percentile latency (ns)": 173083050, "99.00 percentile latency (ns)": 210164310, "99.90 percentile latency (ns)": 229219462, "Max latency (ns)": 254823430, "Mean latency (ns)": 52952633, "Min duration satisfied": "Yes", "Min latency (ns)": 39743381, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 18.87, "QPS w/o loadgen overhead": 18.88, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 69.59761, "characteristics.90th_percentile_latency_ns": 69597610.0, "characteristics.90th_percentile_latency_s": 0.06959761, "characteristics.90th_percentile_latency_us": 69597.61, "characteristics.accuracy": 73.584, "characteristics.good": 36792, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.4-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.4-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_140.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.1818, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "4b9d811282bd1063", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 40584579, "90.00 percentile latency (ns)": 136763448, "90th percentile latency (ns)": 136763448, "95.00 percentile latency (ns)": 178781212, "97.00 percentile latency (ns)": 185917151, "99.00 percentile latency (ns)": 194406141, "99.90 percentile latency (ns)": 206307694, "Max latency (ns)": 212562703, "Mean latency (ns)": 61550240, "Min duration satisfied": "Yes", "Min latency (ns)": 37729919, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.24, "QPS w/o loadgen overhead": 16.25, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 136.763448, "characteristics.90th_percentile_latency_ns": 136763448.0, "characteristics.90th_percentile_latency_s": 0.136763448, "characteristics.90th_percentile_latency_us": 136763.448, "characteristics.accuracy": 72.678, "characteristics.good": 36339, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 13.5135, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "7a191453b9e65f46", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20320465, "90.00 percentile latency (ns)": 37608561, "90th percentile latency (ns)": 37608561, "95.00 percentile latency (ns)": 62329245, "97.00 percentile latency (ns)": 113245349, "99.00 percentile latency (ns)": 171833717, "99.90 percentile latency (ns)": 186307213, "Max latency (ns)": 204630543, "Mean latency (ns)": 27927407, "Min duration satisfied": "Yes", "Min latency (ns)": 19294991, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 35.75, "QPS w/o loadgen overhead": 35.81, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 37.608561, "characteristics.90th_percentile_latency_ns": 37608561.0, "characteristics.90th_percentile_latency_s": 0.037608561, "characteristics.90th_percentile_latency_us": 37608.561, "characteristics.accuracy": 70.43, "characteristics.good": 35215, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 31.25, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "34a411626c734879", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 40724219, "90.00 percentile latency (ns)": 142885348, "90th percentile latency (ns)": 142885348, "95.00 percentile latency (ns)": 178986945, "97.00 percentile latency (ns)": 184916233, "99.00 percentile latency (ns)": 192241144, "99.90 percentile latency (ns)": 203334135, "Max latency (ns)": 211794462, "Mean latency (ns)": 62287004, "Min duration satisfied": "Yes", "Min latency (ns)": 36586638, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.05, "QPS w/o loadgen overhead": 16.05, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 142.885348, "characteristics.90th_percentile_latency_ns": 142885348.0, "characteristics.90th_percentile_latency_s": 0.142885348, "characteristics.90th_percentile_latency_us": 142885.348, "characteristics.accuracy": 73.724, "characteristics.good": 36862, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.0482, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "16b427aa9af8b508", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 47920505, "90.00 percentile latency (ns)": 148144554, "90th percentile latency (ns)": 148144554, "95.00 percentile latency (ns)": 187204362, "97.00 percentile latency (ns)": 195508978, "99.00 percentile latency (ns)": 204716307, "99.90 percentile latency (ns)": 216698547, "Max latency (ns)": 230300413, "Mean latency (ns)": 71103859, "Min duration satisfied": "Yes", "Min latency (ns)": 46066091, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 14.06, "QPS w/o loadgen overhead": 14.06, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 148.144554, "characteristics.90th_percentile_latency_ns": 148144554.0, "characteristics.90th_percentile_latency_s": 0.148144554, "characteristics.90th_percentile_latency_us": 148144.554, "characteristics.accuracy": 75.76, "characteristics.good": 37880, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 13.3333, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "76631794cb9810cd", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 28343272, "90.00 percentile latency (ns)": 54925460, "90th percentile latency (ns)": 54925460, "95.00 percentile latency (ns)": 105130884, "97.00 percentile latency (ns)": 156958972, "99.00 percentile latency (ns)": 185274709, "99.90 percentile latency (ns)": 199068777, "Max latency (ns)": 206972320, "Mean latency (ns)": 39074021, "Min duration satisfied": "Yes", "Min latency (ns)": 27387188, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 25.56, "QPS w/o loadgen overhead": 25.59, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 54.92546, "characteristics.90th_percentile_latency_ns": 54925460.0, "characteristics.90th_percentile_latency_s": 0.05492546, "characteristics.90th_percentile_latency_us": 54925.46, "characteristics.accuracy": 72.91, "characteristics.good": 36455, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 27.027, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "7d914baf7b2673fb", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 22166363, "90.00 percentile latency (ns)": 118090833, "90th percentile latency (ns)": 118090833, "95.00 percentile latency (ns)": 125432297, "97.00 percentile latency (ns)": 128582941, "99.00 percentile latency (ns)": 133061336, "99.90 percentile latency (ns)": 141411892, "Max latency (ns)": 151787647, "Mean latency (ns)": 43672034, "Min duration satisfied": "Yes", "Min latency (ns)": 15316868, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 22.88, "QPS w/o loadgen overhead": 22.9, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 118.090833, "characteristics.90th_percentile_latency_ns": 118090833.0, "characteristics.90th_percentile_latency_s": 0.118090833, "characteristics.90th_percentile_latency_us": 118090.833, "characteristics.accuracy": 63.33, "characteristics.good": 31665, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small-minimalistic_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small-minimalistic_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.1515, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "c93ee6c940cf85bc", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20941283, "90.00 percentile latency (ns)": 120124191, "90th percentile latency (ns)": 120124191, "95.00 percentile latency (ns)": 128274408, "97.00 percentile latency (ns)": 131334281, "99.00 percentile latency (ns)": 136386437, "99.90 percentile latency (ns)": 143850135, "Max latency (ns)": 148640802, "Mean latency (ns)": 42570811, "Min duration satisfied": "Yes", "Min latency (ns)": 16920487, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 23.47, "QPS w/o loadgen overhead": 23.49, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 120.124191, "characteristics.90th_percentile_latency_ns": 120124191.0, "characteristics.90th_percentile_latency_s": 0.120124191, "characteristics.90th_percentile_latency_us": 120124.191, "characteristics.accuracy": 66.216, "characteristics.good": 33108, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 15.3846, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "5576f49e1ae0d4c4", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20471162, "90.00 percentile latency (ns)": 90984416, "90th percentile latency (ns)": 90984416, "95.00 percentile latency (ns)": 126942960, "97.00 percentile latency (ns)": 131793943, "99.00 percentile latency (ns)": 137545287, "99.90 percentile latency (ns)": 146154368, "Max latency (ns)": 153606213, "Mean latency (ns)": 34993742, "Min duration satisfied": "Yes", "Min latency (ns)": 19234206, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 28.55, "QPS w/o loadgen overhead": 28.58, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 90.984416, "characteristics.90th_percentile_latency_ns": 90984416.0, "characteristics.90th_percentile_latency_s": 0.090984416, "characteristics.90th_percentile_latency_us": 90984.416, "characteristics.accuracy": 68.336, "characteristics.good": 34168, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 19.6078, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "74aa6b14ef1b2597", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 12214715, "90.00 percentile latency (ns)": 20767838, "90th percentile latency (ns)": 20767838, "95.00 percentile latency (ns)": 37794467, "97.00 percentile latency (ns)": 75555898, "99.00 percentile latency (ns)": 120644553, "99.90 percentile latency (ns)": 133118886, "Max latency (ns)": 145581929, "Mean latency (ns)": 17070558, "Min duration satisfied": "Yes", "Min latency (ns)": 11446458, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 58.37, "QPS w/o loadgen overhead": 58.58, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 20.767838, "characteristics.90th_percentile_latency_ns": 20767838.0, "characteristics.90th_percentile_latency_s": 0.020767838, "characteristics.90th_percentile_latency_us": 20767.838, "characteristics.accuracy": 64.486, "characteristics.good": 32243, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-small_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-small_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-small_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 41.6667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "30e42c099d263f09", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 460391961, "90.00 percentile latency (ns)": 513365769, "90th percentile latency (ns)": 513365769, "95.00 percentile latency (ns)": 540086550, "97.00 percentile latency (ns)": 566083793, "99.00 percentile latency (ns)": 599433317, "99.90 percentile latency (ns)": 629419677, "Max latency (ns)": 636100157, "Mean latency (ns)": 468588937, "Min duration satisfied": "Yes", "Min latency (ns)": 410442585, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.13, "QPS w/o loadgen overhead": 2.13, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 513.365769, "characteristics.90th_percentile_latency_ns": 513365769.0, "characteristics.90th_percentile_latency_s": 0.513365769, "characteristics.90th_percentile_latency_us": 513365.769, "characteristics.accuracy": 76.442, "characteristics.good": 38221, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "closed", "formal_model": "resnet50-v1.5", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "resnet50", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/closed/Krai/code", "note_details": "/tree/master/closed/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://www.dropbox.com/s/vhuqo0wc39lky0a/resnet50_v1.no-argmax.tflite", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "cf9fe3d32d14695b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TF -> TFLite" } ]