[ { "50.00 percentile latency (ns)": 2730588, "90.00 percentile latency (ns)": 2926880, "90th percentile latency (ns)": 2926880, "95.00 percentile latency (ns)": 3304880, "97.00 percentile latency (ns)": 4201173, "99.00 percentile latency (ns)": 10234308, "99.90 percentile latency (ns)": 76645992, "Max latency (ns)": 97319692, "Mean latency (ns)": 3024682, "Min duration satisfied": "Yes", "Min latency (ns)": 2192462, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 326.77, "QPS w/o loadgen overhead": 330.61, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 42.384, "characteristics.good": 21192, "characteristics.samples_per_second": 330.61, "characteristics.samples_per_second.normalized_per_core": 55.10166666666667, "characteristics.samples_per_second.normalized_per_processor": 330.61, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 250, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "d8a409683fde44b2", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3665669, "90.00 percentile latency (ns)": 3910669, "90th percentile latency (ns)": 3910669, "95.00 percentile latency (ns)": 4677169, "97.00 percentile latency (ns)": 5617795, "99.00 percentile latency (ns)": 18406510, "99.90 percentile latency (ns)": 106049478, "Max latency (ns)": 168409023, "Mean latency (ns)": 4138556, "Min duration satisfied": "Yes", "Min latency (ns)": 3153794, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 239.27, "QPS w/o loadgen overhead": 241.63, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 43.468, "characteristics.good": 21734, "characteristics.samples_per_second": 241.63, "characteristics.samples_per_second.normalized_per_core": 40.27166666666667, "characteristics.samples_per_second.normalized_per_processor": 241.63, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.35-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.35-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_35.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "69e1a914a882ce61", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 3958506, "90.00 percentile latency (ns)": 4203799, "90th percentile latency (ns)": 4203799, "95.00 percentile latency (ns)": 5117884, "97.00 percentile latency (ns)": 5860760, "99.00 percentile latency (ns)": 20880161, "99.90 percentile latency (ns)": 82237296, "Max latency (ns)": 105842499, "Mean latency (ns)": 4395911, "Min duration satisfied": "Yes", "Min latency (ns)": 3143880, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 225.98, "QPS w/o loadgen overhead": 227.48, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 45.494, "characteristics.good": 22747, "characteristics.samples_per_second": 227.48, "characteristics.samples_per_second.normalized_per_core": 37.913333333333334, "characteristics.samples_per_second.normalized_per_processor": 227.48, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.25-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.25-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.25_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 200, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "82483880c79fde28", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4536301, "90.00 percentile latency (ns)": 4827087, "90th percentile latency (ns)": 4827087, "95.00 percentile latency (ns)": 5834796, "97.00 percentile latency (ns)": 7165379, "99.00 percentile latency (ns)": 25497262, "99.90 percentile latency (ns)": 129616094, "Max latency (ns)": 177859017, "Mean latency (ns)": 5248003, "Min duration satisfied": "Yes", "Min latency (ns)": 4066717, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 189.23, "QPS w/o loadgen overhead": 190.55, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 49.11, "characteristics.good": 24555, "characteristics.samples_per_second": 190.55, "characteristics.samples_per_second.normalized_per_core": 31.758333333333336, "characteristics.samples_per_second.normalized_per_processor": 190.55, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.5-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.5-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_50.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 166.667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "184677accb6a0d35", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 4758237, "90.00 percentile latency (ns)": 5070901, "90th percentile latency (ns)": 5070901, "95.00 percentile latency (ns)": 6425106, "97.00 percentile latency (ns)": 7324606, "99.00 percentile latency (ns)": 30748539, "99.90 percentile latency (ns)": 92909809, "Max latency (ns)": 110418255, "Mean latency (ns)": 5556298, "Min duration satisfied": "Yes", "Min latency (ns)": 4188612, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 178.91, "QPS w/o loadgen overhead": 179.98, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 56.75, "characteristics.good": 28375, "characteristics.samples_per_second": 179.98, "characteristics.samples_per_second.normalized_per_core": 29.996666666666666, "characteristics.samples_per_second.normalized_per_processor": 179.98, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 142.857, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "867099e403f99ee9", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6363308, "90.00 percentile latency (ns)": 6912225, "90th percentile latency (ns)": 6912225, "95.00 percentile latency (ns)": 8768984, "97.00 percentile latency (ns)": 12195198, "99.00 percentile latency (ns)": 35451960, "99.90 percentile latency (ns)": 160676373, "Max latency (ns)": 190830690, "Mean latency (ns)": 7541181, "Min duration satisfied": "Yes", "Min latency (ns)": 5678180, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 131.99, "QPS w/o loadgen overhead": 132.61, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 57.002, "characteristics.good": 28501, "characteristics.samples_per_second": 132.61, "characteristics.samples_per_second.normalized_per_core": 22.10166666666667, "characteristics.samples_per_second.normalized_per_processor": 132.61, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-96-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-96-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_96_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 125, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "17b315f5f3540e69", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 6887138, "90.00 percentile latency (ns)": 7588599, "90th percentile latency (ns)": 7588599, "95.00 percentile latency (ns)": 9032935, "97.00 percentile latency (ns)": 13319692, "99.00 percentile latency (ns)": 45967515, "99.90 percentile latency (ns)": 99540194, "Max latency (ns)": 111103213, "Mean latency (ns)": 8016714, "Min duration satisfied": "Yes", "Min latency (ns)": 6044803, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 124.25, "QPS w/o loadgen overhead": 124.74, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 59.444, "characteristics.good": 29722, "characteristics.samples_per_second": 124.74, "characteristics.samples_per_second.normalized_per_core": 20.79, "characteristics.samples_per_second.normalized_per_processor": 124.74, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 71.4286, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "1253c5395442fa09", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 8911875, "90.00 percentile latency (ns)": 9939708, "90th percentile latency (ns)": 9939708, "95.00 percentile latency (ns)": 12300745, "97.00 percentile latency (ns)": 22090457, "99.00 percentile latency (ns)": 63948113, "99.90 percentile latency (ns)": 103096118, "Max latency (ns)": 117359647, "Mean latency (ns)": 10254123, "Min duration satisfied": "Yes", "Min latency (ns)": 7831250, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 97.22, "QPS w/o loadgen overhead": 97.52, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 61.456, "characteristics.good": 30728, "characteristics.samples_per_second": 97.52, "characteristics.samples_per_second.normalized_per_core": 16.253333333333334, "characteristics.samples_per_second.normalized_per_processor": 97.52, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.5-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.5-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.5_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "197f32f43638e7ec", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 9345883, "90.00 percentile latency (ns)": 10262592, "90th percentile latency (ns)": 10262592, "95.00 percentile latency (ns)": 13389261, "97.00 percentile latency (ns)": 23671271, "99.00 percentile latency (ns)": 52784919, "99.90 percentile latency (ns)": 169222984, "Max latency (ns)": 190353819, "Mean latency (ns)": 11014115, "Min duration satisfied": "Yes", "Min latency (ns)": 8397382, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 90.52, "QPS w/o loadgen overhead": 90.79, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 62.07, "characteristics.good": 31035, "characteristics.samples_per_second": 90.79, "characteristics.samples_per_second.normalized_per_core": 15.131666666666668, "characteristics.samples_per_second.normalized_per_processor": 90.79, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_128_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 83.3333, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "295482d62eec0e92", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11227419, "90.00 percentile latency (ns)": 13519919, "90th percentile latency (ns)": 13519919, "95.00 percentile latency (ns)": 21320382, "97.00 percentile latency (ns)": 32095176, "99.00 percentile latency (ns)": 84510013, "99.90 percentile latency (ns)": 107174909, "Max latency (ns)": 121204556, "Mean latency (ns)": 13308576, "Min duration satisfied": "Yes", "Min latency (ns)": 10308968, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 74.95, "QPS w/o loadgen overhead": 75.14, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.08, "characteristics.good": 32040, "characteristics.samples_per_second": 75.14, "characteristics.samples_per_second.normalized_per_core": 12.523333333333333, "characteristics.samples_per_second.normalized_per_processor": 75.14, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "34c1293bf2e5b900", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 11452855, "90.00 percentile latency (ns)": 13839854, "90th percentile latency (ns)": 13839854, "95.00 percentile latency (ns)": 21852507, "97.00 percentile latency (ns)": 32943390, "99.00 percentile latency (ns)": 82594100, "99.90 percentile latency (ns)": 108376746, "Max latency (ns)": 121321486, "Mean latency (ns)": 13625916, "Min duration satisfied": "Yes", "Min latency (ns)": 10658358, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 73.18, "QPS w/o loadgen overhead": 73.39, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.768, "characteristics.good": 32384, "characteristics.samples_per_second": 73.39, "characteristics.samples_per_second.normalized_per_core": 12.231666666666667, "characteristics.samples_per_second.normalized_per_processor": 73.39, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-128-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-128-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_128_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 66.6667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "d6f4067ab813617d", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 13515873, "90.00 percentile latency (ns)": 15632213, "90th percentile latency (ns)": 15632213, "95.00 percentile latency (ns)": 26927257, "97.00 percentile latency (ns)": 38393193, "99.00 percentile latency (ns)": 78908592, "99.90 percentile latency (ns)": 182317462, "Max latency (ns)": 200299919, "Mean latency (ns)": 15952584, "Min duration satisfied": "Yes", "Min latency (ns)": 12071554, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 62.56, "QPS w/o loadgen overhead": 62.69, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 64.972, "characteristics.good": 32486, "characteristics.samples_per_second": 62.69, "characteristics.samples_per_second.normalized_per_core": 10.448333333333332, "characteristics.samples_per_second.normalized_per_processor": 62.69, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-0.75-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-0.75-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_160_75.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 41.6667, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "bab91362925e40df", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 14653339, "90.00 percentile latency (ns)": 18213715, "90th percentile latency (ns)": 18213715, "95.00 percentile latency (ns)": 32555846, "97.00 percentile latency (ns)": 44771434, "99.00 percentile latency (ns)": 99529581, "99.90 percentile latency (ns)": 116789545, "Max latency (ns)": 137581845, "Mean latency (ns)": 17458367, "Min duration satisfied": "Yes", "Min latency (ns)": 13464214, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 57.15, "QPS w/o loadgen overhead": 57.28, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 67.24, "characteristics.good": 33620, "characteristics.samples_per_second": 57.28, "characteristics.samples_per_second.normalized_per_core": 9.546666666666667, "characteristics.samples_per_second.normalized_per_processor": 57.28, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-0.75-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-0.75-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_0.75_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 55.5556, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "71ca5d40bcaf253b", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 16706711, "90.00 percentile latency (ns)": 20832059, "90th percentile latency (ns)": 20832059, "95.00 percentile latency (ns)": 34428515, "97.00 percentile latency (ns)": 45665545, "99.00 percentile latency (ns)": 98353047, "99.90 percentile latency (ns)": 116337492, "Max latency (ns)": 134855826, "Mean latency (ns)": 19576610, "Min duration satisfied": "Yes", "Min latency (ns)": 15677421, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 50.99, "QPS w/o loadgen overhead": 51.08, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 68.248, "characteristics.good": 34124, "characteristics.samples_per_second": 51.08, "characteristics.samples_per_second.normalized_per_core": 8.513333333333334, "characteristics.samples_per_second.normalized_per_processor": 51.08, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-160-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-160-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_160_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 47.619, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "bd59dc4b574f43a9", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 21975023, "90.00 percentile latency (ns)": 29024868, "90th percentile latency (ns)": 29024868, "95.00 percentile latency (ns)": 44865639, "97.00 percentile latency (ns)": 61585670, "99.00 percentile latency (ns)": 148401649, "99.90 percentile latency (ns)": 196581733, "Max latency (ns)": 219009179, "Mean latency (ns)": 25987340, "Min duration satisfied": "Yes", "Min latency (ns)": 19767113, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 38.42, "QPS w/o loadgen overhead": 38.48, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 69.604, "characteristics.good": 34802, "characteristics.samples_per_second": 38.48, "characteristics.samples_per_second.normalized_per_core": 6.413333333333333, "characteristics.samples_per_second.normalized_per_processor": 38.48, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_192_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 35.7143, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "1bd2d5c8224415bb", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 22883863, "90.00 percentile latency (ns)": 31358527, "90th percentile latency (ns)": 31358527, "95.00 percentile latency (ns)": 46005227, "97.00 percentile latency (ns)": 62535262, "99.00 percentile latency (ns)": 112243495, "99.90 percentile latency (ns)": 125623398, "Max latency (ns)": 135226530, "Mean latency (ns)": 26546330, "Min duration satisfied": "Yes", "Min latency (ns)": 21253238, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 37.61, "QPS w/o loadgen overhead": 37.67, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.04, "characteristics.good": 35020, "characteristics.samples_per_second": 37.67, "characteristics.samples_per_second.normalized_per_core": 6.278333333333333, "characteristics.samples_per_second.normalized_per_processor": 37.67, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-192-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-192-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_192_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 37.037, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "1fb422787059c8f9", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 20320465, "90.00 percentile latency (ns)": 37608561, "90th percentile latency (ns)": 37608561, "95.00 percentile latency (ns)": 62329245, "97.00 percentile latency (ns)": 113245349, "99.00 percentile latency (ns)": 171833717, "99.90 percentile latency (ns)": 186307213, "Max latency (ns)": 204630543, "Mean latency (ns)": 27927407, "Min duration satisfied": "Yes", "Min latency (ns)": 19294991, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 35.75, "QPS w/o loadgen overhead": 35.81, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.43, "characteristics.good": 35215, "characteristics.samples_per_second": 35.81, "characteristics.samples_per_second.normalized_per_core": 5.968333333333334, "characteristics.samples_per_second.normalized_per_processor": 35.81, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large-minimalistic_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large-minimalistic_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 31.25, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "cb9ee3f53faad8e3", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 26494628, "90.00 percentile latency (ns)": 49034181, "90th percentile latency (ns)": 49034181, "95.00 percentile latency (ns)": 92485159, "97.00 percentile latency (ns)": 155239706, "99.00 percentile latency (ns)": 194591374, "99.90 percentile latency (ns)": 208654179, "Max latency (ns)": 222012130, "Mean latency (ns)": 36554990, "Min duration satisfied": "Yes", "Min latency (ns)": 25653508, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 27.32, "QPS w/o loadgen overhead": 27.36, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.542, "characteristics.good": 35271, "characteristics.samples_per_second": 27.36, "characteristics.samples_per_second.normalized_per_core": 4.56, "characteristics.samples_per_second.normalized_per_processor": 27.36, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_100.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 25.641, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "c9bac43fdb49642b", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 29471510, "90.00 percentile latency (ns)": 59180448, "90th percentile latency (ns)": 59180448, "95.00 percentile latency (ns)": 111633316, "97.00 percentile latency (ns)": 120648137, "99.00 percentile latency (ns)": 127563838, "99.90 percentile latency (ns)": 138155778, "Max latency (ns)": 144373010, "Mean latency (ns)": 38957607, "Min duration satisfied": "Yes", "Min latency (ns)": 28517759, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 25.62, "QPS w/o loadgen overhead": 25.67, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 70.772, "characteristics.good": 35386, "characteristics.samples_per_second": 25.67, "characteristics.samples_per_second.normalized_per_core": 4.278333333333333, "characteristics.samples_per_second.normalized_per_processor": 25.67, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v1-1.0-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v1-1.0-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224_quant.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 24.3902, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "4048f4b892174764", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 28343272, "90.00 percentile latency (ns)": 54925460, "90th percentile latency (ns)": 54925460, "95.00 percentile latency (ns)": 105130884, "97.00 percentile latency (ns)": 156958972, "99.00 percentile latency (ns)": 185274709, "99.90 percentile latency (ns)": 199068777, "Max latency (ns)": 206972320, "Mean latency (ns)": 39074021, "Min duration satisfied": "Yes", "Min latency (ns)": 27387188, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 25.56, "QPS w/o loadgen overhead": 25.59, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 72.91, "characteristics.good": 36455, "characteristics.samples_per_second": 25.59, "characteristics.samples_per_second.normalized_per_core": 4.265, "characteristics.samples_per_second.normalized_per_processor": 25.59, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_uint8", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_1.0_uint8", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_uint8.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 27.027, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "ddb8f97c994c17bb", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 40968398, "90.00 percentile latency (ns)": 69597610, "90th percentile latency (ns)": 69597610, "95.00 percentile latency (ns)": 123156810, "97.00 percentile latency (ns)": 173083050, "99.00 percentile latency (ns)": 210164310, "99.90 percentile latency (ns)": 229219462, "Max latency (ns)": 254823430, "Mean latency (ns)": 52952633, "Min duration satisfied": "Yes", "Min latency (ns)": 39743381, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 18.87, "QPS w/o loadgen overhead": 18.88, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.584, "characteristics.good": 36792, "characteristics.samples_per_second": 18.88, "characteristics.samples_per_second.normalized_per_core": 3.1466666666666665, "characteristics.samples_per_second.normalized_per_processor": 18.88, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v2-1.4-224-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v2-1.4-224-quantized", "input_data_types": "int8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v2/checkpoints/quantized_v2_224_140.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 18.1818, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "447c7b4c3953cadc", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 40724219, "90.00 percentile latency (ns)": 142885348, "90th percentile latency (ns)": 142885348, "95.00 percentile latency (ns)": 178986945, "97.00 percentile latency (ns)": 184916233, "99.00 percentile latency (ns)": 192241144, "99.90 percentile latency (ns)": 203334135, "Max latency (ns)": 211794462, "Mean latency (ns)": 62287004, "Min duration satisfied": "Yes", "Min latency (ns)": 36586638, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.05, "QPS w/o loadgen overhead": 16.05, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 73.724, "characteristics.good": 36862, "characteristics.samples_per_second": 16.05, "characteristics.samples_per_second.normalized_per_core": 2.6750000000000003, "characteristics.samples_per_second.normalized_per_processor": 16.05, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_0.75_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_0.75_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_0.75_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 12.0482, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "f8bcd872a07eb2ff", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 47920505, "90.00 percentile latency (ns)": 148144554, "90th percentile latency (ns)": 148144554, "95.00 percentile latency (ns)": 187204362, "97.00 percentile latency (ns)": 195508978, "99.00 percentile latency (ns)": 204716307, "99.90 percentile latency (ns)": 216698547, "Max latency (ns)": 230300413, "Mean latency (ns)": 71103859, "Min duration satisfied": "Yes", "Min latency (ns)": 46066091, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 14.06, "QPS w/o loadgen overhead": 14.06, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 75.76, "characteristics.good": 37880, "characteristics.samples_per_second": 14.06, "characteristics.samples_per_second.normalized_per_core": 2.3433333333333333, "characteristics.samples_per_second.normalized_per_processor": 14.06, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "mobilenet-v3-large_224_1.0_float", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "mobilenet-v3-large_224_1.0_float", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/mobilenet_v3/checkpoints/v3-large_224_1.0_float.tgz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 13.3333, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "bb4a8ab5420b5f58", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 106646431, "90.00 percentile latency (ns)": 188719818, "90th percentile latency (ns)": 188719818, "95.00 percentile latency (ns)": 251657516, "97.00 percentile latency (ns)": 269446840, "99.00 percentile latency (ns)": 285280972, "99.90 percentile latency (ns)": 298914223, "Max latency (ns)": 318689039, "Mean latency (ns)": 122777928, "Min duration satisfied": "Yes", "Min latency (ns)": 92435266, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 8.14, "QPS w/o loadgen overhead": 8.14, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 76.642, "characteristics.good": 38321, "characteristics.samples_per_second": 8.14, "characteristics.samples_per_second.normalized_per_core": 1.3566666666666667, "characteristics.samples_per_second.normalized_per_processor": 8.14, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite1-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite1-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite1.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 7.69231, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "cb0092979476c90a", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 235354940, "90.00 percentile latency (ns)": 293672880, "90th percentile latency (ns)": 293672880, "95.00 percentile latency (ns)": 303602443, "97.00 percentile latency (ns)": 309512456, "99.00 percentile latency (ns)": 317951459, "99.90 percentile latency (ns)": 341667413, "Max latency (ns)": 448119929, "Mean latency (ns)": 226369438, "Min duration satisfied": "Yes", "Min latency (ns)": 123688929, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 4.42, "QPS w/o loadgen overhead": 4.42, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 77.542, "characteristics.good": 38771, "characteristics.samples_per_second": 4.42, "characteristics.samples_per_second.normalized_per_core": 0.7366666666666667, "characteristics.samples_per_second.normalized_per_processor": 4.42, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite2-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite2-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite2.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 4.1841, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "5c314476b187507e", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 324260412, "90.00 percentile latency (ns)": 416209128, "90th percentile latency (ns)": 416209128, "95.00 percentile latency (ns)": 433856470, "97.00 percentile latency (ns)": 440011178, "99.00 percentile latency (ns)": 455279870, "99.90 percentile latency (ns)": 473264667, "Max latency (ns)": 476895076, "Mean latency (ns)": 305669961, "Min duration satisfied": "Yes", "Min latency (ns)": 178695293, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 3.27, "QPS w/o loadgen overhead": 3.27, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 78.514, "characteristics.good": 39257, "characteristics.samples_per_second": 3.27, "characteristics.samples_per_second.normalized_per_core": 0.545, "characteristics.samples_per_second.normalized_per_processor": 3.27, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite4-quantized", "input_data_types": "uint8", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.3557, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "f24d5bb681d95d15", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 328586430, "90.00 percentile latency (ns)": 380736463, "90th percentile latency (ns)": 380736463, "95.00 percentile latency (ns)": 390845357, "97.00 percentile latency (ns)": 396542383, "99.00 percentile latency (ns)": 409008172, "99.90 percentile latency (ns)": 430273817, "Max latency (ns)": 436923714, "Mean latency (ns)": 324454976, "Min duration satisfied": "Yes", "Min latency (ns)": 183151052, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 3.08, "QPS w/o loadgen overhead": 3.08, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 79.268, "characteristics.good": 39634, "characteristics.samples_per_second": 3.08, "characteristics.samples_per_second.normalized_per_core": 0.5133333333333333, "characteristics.samples_per_second.normalized_per_processor": 3.08, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite3-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite3-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite3.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 3.0581, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "520b45338bb8ef1b", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" }, { "50.00 percentile latency (ns)": 493166225, "90.00 percentile latency (ns)": 545062465, "90th percentile latency (ns)": 545062465, "95.00 percentile latency (ns)": 558331316, "97.00 percentile latency (ns)": 563402405, "99.00 percentile latency (ns)": 578899309, "99.90 percentile latency (ns)": 603919584, "Max latency (ns)": 606813127, "Mean latency (ns)": 486407759, "Min duration satisfied": "Yes", "Min latency (ns)": 317799355, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.06, "QPS w/o loadgen overhead": 2.06, "Result is": "VALID", "SUT name": "TFLite_SUT", "Scenario": "singlestream", "accelerator_frequency": "-", "accelerator_host_interconnect": "-", "accelerator_interconnect": "-", "accelerator_interconnect_topology": "-", "accelerator_memory_capacity": "-", "accelerator_memory_configuration": "-", "accelerator_model_name": "-", "accelerator_on-chip_memories": "-", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.accuracy": 80.676, "characteristics.good": 40338, "characteristics.samples_per_second": 2.06, "characteristics.samples_per_second.normalized_per_core": 0.3433333333333333, "characteristics.samples_per_second.normalized_per_processor": 2.06, "characteristics.total": 50000, "ck_system": "firefly-tflite-v2.5.0-ruy", "ck_used": true, "cooling": "on-board fan", "dataset": "ImageNet 2012", "dataset_link": "https://github.com/ctuning/ck/blob/master/docs/mlperf-automation/datasets/imagenet2012.md", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.accuracy", "dim_y_maximize": true, "division": "open", "formal_model": "efficientnet-lite4-non-quantized", "formal_model_accuracy": 99.0, "formal_model_link": "https://github.com/octoml/mlops/tree/main/package, https://github.com/ctuning/ai/tree/main/package", "framework": "TFLite v2.5.0 (ruy)", "host_memory_capacity": "4 GiB", "host_memory_configuration": "-", "host_networking": "-", "host_networking_topology": "-", "host_processor_caches": "L1I$ 48 KiB, L1D$ 32 KiB, L2$ 1 MiB (big); L1I$ 32 KiB, L1D$ 32 KiB, L2$ 512 KiB (LITTLE)", "host_processor_core_count": 6, "host_processor_frequency": "1800 MHz (big), 1400 MHz (LITTLE)", "host_processor_interconnect": "-", "host_processor_model_name": "Arm Cortex-A72 MP2 (big); Arm Cortex-A53 MP4 (LITTLE)", "host_processors_per_node": 1, "host_storage_capacity": "128 GiB", "host_storage_type": "SanDisk Extreme microSD", "hw_notes": "", "informal_model": "efficientnet-lite4-non-quantized", "input_data_types": "fp32", "key.accuracy": "characteristics.accuracy", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": "", "normalize_cores": 6, "normalize_processors": 1, "note_code": "/tree/master/open/Krai/code", "note_details": "/tree/master/open/Krai/results/firefly-tflite-v2.5.0-ruy", "number_of_nodes": 1, "operating_system": "Ubuntu 16.04.7 LTS; kernel 4.4.77 #554 (Thu Nov 30 11:30:11 HKT 2017)", "other_software_stack": "", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 1024, "print_timestamps": 0, "problem": true, "problem_str": "scenario in meta (singlestream) doesn't match directory (offline)", "qsl_rng_seed": 1624344308455410291, "retraining": "no", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/lite/efficientnet-lite4.tar.gz", "status": "available", "submitter": "Krai", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/Krai", "sw_notes": "Powered by Collective Knowledge v1.17.0", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/firefly-tflite-v2.5.0-ruy", "system_name": "Firefly-RK3399 (firefly)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.04082, "task": "image classification", "task2": "image classification", "total_cores": 6, "uid": "41f3a359ebc55229", "use_accelerator": false, "weight_data_types": "fp32", "weight_transformations": "TFLite" } ]