benchmark/src/main/java/androidx/benchmark/BenchmarkState.kt - platform/frameworks/support - Git at Google

 /*
  * Copyright (C) 2016 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 package androidx.benchmark

 import android.app.Activity
 import android.os.Build
 import android.os.Bundle
 import android.os.Debug
 import android.util.Log
 import androidx.annotation.IntRange
 import androidx.annotation.VisibleForTesting
 import androidx.test.platform.app.InstrumentationRegistry
 import java.io.File
 import java.text.NumberFormat
 import java.util.ArrayList
 import java.util.concurrent.TimeUnit

 /**
  * Control object for benchmarking in the code in Java.
  *
  * Query a state object with [BenchmarkRule.state], and use it to measure a block of Java with
  * [BenchmarkState.keepRunning]:
  * ```
  * @Rule
  * public BenchmarkRule benchmarkRule = new BenchmarkRule();
  *
  * @Test
  * public void sampleMethod() {
  *     BenchmarkState state = benchmarkRule.getState();
  *
  *     int[] src = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
  *     while (state.keepRunning()) {
  *         int[] dest = new int[src.length];
  *         System.arraycopy(src, 0, dest, 0, src.length);
  *     }
  * }
  * ```
  *
  * @see BenchmarkRule#getState()
  */
 class BenchmarkState internal constructor() {
     private var warmupIteration = 0 // increasing iteration count during warmup

     /**
      * Decreasing iteration count used when [state] == [RUNNING], used to determine when main
      * measurement loop finishes.
      */
     @JvmField // Used by [BenchmarkState.keepRunningInline()]
     @PublishedApi
     internal var iterationsRemaining = -1

     private var maxIterations = 0

     private var state = NOT_STARTED // Current benchmark state.

     private val warmupManager = WarmupManager()

     private var startTimeNs: Long = 0 // System.nanoTime() at start of last warmup/test iter.

     private var paused = false
     private var pausedTimeNs: Long = 0 // The System.nanoTime() when the pauseTiming() is called.
     private var pausedDurationNs: Long = 0 // The duration of paused state in nano sec.
     private var thermalThrottleSleepSeconds: Long =
         0 // The duration of sleep due to thermal throttling.

     private var repeatCount = 0

     // Statistics. These values will be filled when the benchmark has finished.
     // The computation needs double precision, but long int is fine for final reporting.
     private var internalStats: Stats? = null

     // Individual duration in nano seconds.
     private val results = ArrayList<Long>()

     internal var performThrottleChecks = true
     private var throttleRemainingRetries = THROTTLE_MAX_RETRIES

     /**
      * Get the end of run benchmark statistics.
      *
      *
      * This method may only be called keepRunning() returns `false`.
      *
      * @return Stats from run.
      */
     internal val stats: Stats
         get() {
             if (state == NOT_STARTED) {
                 throw IllegalStateException(
                     "The benchmark wasn't started! Every test in a class " +
                             "with a BenchmarkRule must contain a benchmark. In Kotlin, call " +
                             "benchmarkRule.measureRepeated {}, or in Java, call " +
                             "benchmarkRule.getState().keepRunning() to run your benchmark."
                 )
             }
             if (state != FINISHED) {
                 throw IllegalStateException(
                     "The benchmark hasn't finished! In Java, use " +
                             "while(BenchmarkState.keepRunning()) to ensure keepRunning() returns " +
                             "false before ending your test. In Kotlin, just use " +
                             "benchmarkRule.measureRepeated {} to avoid the problem."
                 )
             }
             return internalStats!!
         }

     /**
      * Stops the benchmark timer.
      *
      * This method can be called only when the timer is running.
      *
      * ```
      * @Test
      * public void bitmapProcessing() {
      *     final BenchmarkState state = mBenchmarkRule.getState();
      *     while (state.keepRunning()) {
      *         state.pauseTiming();
      *         // disable timing while constructing test input
      *         Bitmap input = constructTestBitmap();
      *         state.resumeTiming();
      *
      *         processBitmap(input);
      *     }
      * }
      * ```
      *
      * @see resumeTiming
      */
     fun pauseTiming() {
         if (paused) {
             throw IllegalStateException(
                 "Unable to pause the benchmark. The benchmark has already paused."
             )
         }
         pausedTimeNs = System.nanoTime()
         paused = true
     }

     /**
      * Resumes the benchmark timer.
      *
      * This method can be called only when the timer is stopped.
      *
      * ```
      * @Test
      * public void bitmapProcessing() {
      *     final BenchmarkState state = mBenchmarkRule.getState();
      *     while (state.keepRunning()) {
      *         state.pauseTiming();
      *         // disable timing while constructing test input
      *         Bitmap input = constructTestBitmap();
      *         state.resumeTiming();
      *
      *         processBitmap(input);
      *     }
      * }
      * ```
      *
      * @see pauseTiming
      */
     fun resumeTiming() {
         if (!paused) {
             throw IllegalStateException(
                 "Unable to resume the benchmark. The benchmark is already running."
             )
         }
         pausedDurationNs += System.nanoTime() - pausedTimeNs
         pausedTimeNs = 0
         paused = false
     }

     private fun beginWarmup() {
         startTimeNs = System.nanoTime()
         warmupIteration = 0
         state = WARMUP
     }

     private fun beginBenchmark() {
         if (ENABLE_PROFILING && Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
             // TODO: support data dir for old platforms
             val f = File(
                 InstrumentationRegistry.getInstrumentation().context.dataDir,
                 "benchprof"
             )
             Log.d(TAG, "Tracing to: " + f.absolutePath)
             Debug.startMethodTracingSampling(f.absolutePath, 16 * 1024 * 1024, 100)
         }
         val idealIterations =
             (TARGET_TEST_DURATION_NS / warmupManager.estimatedIterationTime).toInt()
         maxIterations = Math.min(
             MAX_TEST_ITERATIONS,
             Math.max(idealIterations, MIN_TEST_ITERATIONS)
         )
         pausedDurationNs = 0
         iterationsRemaining = maxIterations
         repeatCount = 0
         thermalThrottleSleepSeconds = 0
         state = RUNNING
         startTimeNs = System.nanoTime()
     }

     private fun startNextTestRun(): Boolean {
         val currentTime = System.nanoTime()

         results.add((currentTime - startTimeNs - pausedDurationNs) / maxIterations)
         repeatCount++

         if (repeatCount >= REPEAT_COUNT) {
             if (performThrottleChecks &&
                 throttleRemainingRetries > 0 &&
                 sleepIfThermalThrottled(THROTTLE_BACKOFF_S)
             ) {
                 // We've slept due to thermal throttle - retry benchmark!
                 throttleRemainingRetries -= 1
                 results.clear()
                 repeatCount = 0
             } else {
                 // finished!
                 if (ENABLE_PROFILING) {
                     Debug.stopMethodTracing()
                 }
                 internalStats = Stats(results)
                 state = FINISHED
                 return false
             }
         }
         pausedDurationNs = 0
         iterationsRemaining = maxIterations
         startTimeNs = System.nanoTime()
         return true
     }

     /**
      * Inline fast-path function for inner benchmark loop.
      *
      * Kotlin users should use [BenchmarkRule.measureRepeated]
      *
      * This codepath uses exclusively @JvmField/const members, so there are no method calls at all
      * in the inlined loop. On recent Android Platform versions, ART inlines these accessors anyway,
      * but we want to be sure it's as simple as possible.
      */
     @Suppress("NOTHING_TO_INLINE")
     @PublishedApi
     internal inline fun keepRunningInline(): Boolean {
         if (iterationsRemaining > 1) {
             iterationsRemaining--
             return true
         }
         return keepRunningInternal()
     }

     /**
      * Returns true if the benchmark needs more samples - use this as the condition of a while loop.
      *
      * ```
      * while (state.keepRunning()) {
      *     int[] dest = new int[src.length];
      *     System.arraycopy(src, 0, dest, 0, src.length);
      * }
      * ```
      */
     fun keepRunning(): Boolean {
         if (iterationsRemaining > 1) {
             iterationsRemaining--
             return true
         }
         return keepRunningInternal()
     }

     @PublishedApi
     internal fun keepRunningInternal(): Boolean {
         when (state) {
             NOT_STARTED -> {
                 if (performThrottleChecks &&
                     !CpuInfo.locked &&
                     !AndroidBenchmarkRunner.sustainedPerformanceModeInUse &&
                     !WarningState.isEmulator
                 ) {
                     ThrottleDetector.computeThrottleBaseline()
                 }
                 beginWarmup()
                 return true
             }
             WARMUP -> {
                 warmupIteration++
                 // Only check nanoTime on every iteration in WARMUP since we
                 // don't yet have a target iteration count.
                 val time = System.nanoTime()
                 val lastDuration = time - startTimeNs
                 startTimeNs = time
                 throwIfPaused() // check each loop during warmup
                 if (warmupManager.onNextIteration(lastDuration)) {
                     beginBenchmark()
                 }
                 return true
             }
             RUNNING -> {
                 iterationsRemaining--
                 if (iterationsRemaining <= 0) {
                     throwIfPaused() // only check at end of loop to save cycles
                     return startNextTestRun()
                 }
                 return true
             }
             FINISHED -> throw IllegalStateException("The benchmark has finished.")
             else -> throw IllegalStateException("The benchmark is in unknown state.")
         }
     }

     private fun throwIfPaused() {
         if (paused) {
             throw IllegalStateException(
                 "Benchmark step finished with paused state." +
                         " Resume the benchmark before finishing each step."
             )
         }
     }

     internal data class Report(
         val className: String,
         val testName: String,
         val data: List<Long>,
         val repeatIterations: Int,
         val thermalThrottleSleepSeconds: Long,
         val warmupIterations: Int
     ) {
         val stats = Stats(data)
     }

     internal fun getReport(testName: String, className: String) = Report(
         className = className,
         testName = testName,
         data = results,
         repeatIterations = maxIterations,
         thermalThrottleSleepSeconds = thermalThrottleSleepSeconds,
         warmupIterations = warmupIteration
     )

     private fun summaryLine() = "Summary: " +
             "median=${stats.median}ns, " +
             "mean=${stats.mean.toLong()}ns, " +
             "min=${stats.min}ns, " +
             "stddev=${stats.standardDeviation.toLong()}ns, " +
             "count=$maxIterations"

     /**
      * Acquires a status report bundle
      *
      * @param key Run identifier, prepended to bundle properties.
      */
     internal fun getFullStatusReport(key: String): Bundle {
         Log.i(TAG, key + summaryLine())
         val status = Bundle()

         val prefix = WarningState.WARNING_PREFIX
         status.putLong("${prefix}median", stats.median)
         status.putLong("${prefix}mean", stats.mean.toLong())
         status.putLong("${prefix}min", stats.min)
         status.putLong("${prefix}standardDeviation", stats.standardDeviation.toLong())
         status.putLong("${prefix}count", maxIterations.toLong())
         status.putIdeSummaryLine(key, stats.min)
         return status
     }

     internal fun sendStatus(testName: String) {
         val bundle = getFullStatusReport(testName)
         InstrumentationRegistry.getInstrumentation().sendStatus(Activity.RESULT_OK, bundle)
     }

     private fun sleepIfThermalThrottled(sleepSeconds: Long) = when {
         ThrottleDetector.isDeviceThermalThrottled() -> {
             Log.d(TAG, "THERMAL THROTTLE DETECTED, SLEEPING FOR $sleepSeconds SECONDS")
             val startTime = System.nanoTime()
             Thread.sleep(TimeUnit.SECONDS.toMillis(sleepSeconds))
             val sleepTime = System.nanoTime() - startTime
             thermalThrottleSleepSeconds += TimeUnit.NANOSECONDS.toSeconds(sleepTime)
             true
         }
         else -> false
     }

     internal companion object {
         private const val TAG = "Benchmark"
         private const val STUDIO_OUTPUT_KEY_PREFIX = "android.studio.display."
         private const val STUDIO_OUTPUT_KEY_ID = "benchmark"

         private const val ENABLE_PROFILING = false

         private const val NOT_STARTED = 0 // The benchmark has not started yet.
         private const val WARMUP = 1 // The benchmark is warming up.
         private const val RUNNING = 2 // The benchmark is running.
         private const val FINISHED = 3 // The benchmark has stopped.

         // Values determined empirically.
         @VisibleForTesting
         internal const val REPEAT_COUNT = 50
         private val TARGET_TEST_DURATION_NS = TimeUnit.MICROSECONDS.toNanos(500)
         private const val MAX_TEST_ITERATIONS = 1000000
         private const val MIN_TEST_ITERATIONS = 1

         private const val THROTTLE_MAX_RETRIES = 3
         private const val THROTTLE_BACKOFF_S = 90L

         /**
          * Hooks for benchmarks not using [BenchmarkRule] to register results.
          *
          * Results are printed to Studio console, and added to the output JSON file.
          *
          * @param className Name of class the benchmark runs in
          * @param testName Name of the benchmark
          * @param dataNs List of all measured results, in nanoseconds
          * @param warmupIterations Number of iterations of warmup before measurements started.
          *                         Should be no less than 0.
          * @param thermalThrottleSleepSeconds Number of seconds benchmark was paused during thermal
          *                                    throttling.
          * @param repeatIterations Number of iterations in between each measurement. Should be no
          *                         less than 1.
          */
         @Suppress("unused")
         @JvmStatic
         fun reportData(
             className: String,
             testName: String,
             dataNs: List<Long>,
             @IntRange(from = 0) warmupIterations: Int,
             @IntRange(from = 0) thermalThrottleSleepSeconds: Long,
             @IntRange(from = 1) repeatIterations: Int
         ) {
             val report = Report(
                 className = className,
                 testName = testName,
                 data = dataNs,
                 repeatIterations = repeatIterations,
                 thermalThrottleSleepSeconds = thermalThrottleSleepSeconds,
                 warmupIterations = warmupIterations
             )

             // Report value to Studio console
             val bundle = Bundle()
             val fullTestName = WarningState.WARNING_PREFIX +
                     if (className.isNotEmpty()) "$className.$testName" else testName
             bundle.putIdeSummaryLine(fullTestName, report.stats.min)
             InstrumentationRegistry.getInstrumentation().sendStatus(Activity.RESULT_OK, bundle)

             // Report values to file output
             ResultWriter.appendReport(report)
         }

         internal fun ideSummaryLineWrapped(key: String, nanos: Long): String {
             val warningLines =
                 WarningState.acquireWarningStringForLogging()?.split("\n") ?: listOf()
             return (warningLines + ideSummaryLine(key, nanos))
                 // remove first line if empty
                 .filterIndexed { index, it -> index != 0 || !it.isEmpty() }
                 // join, prepending key to everything but first string,
                 // to make each line look the same
                 .joinToString("\n$STUDIO_OUTPUT_KEY_ID: ")
         }

         // NOTE: this summary line will use default locale to determine separators. As
         // this line is only meant for human eyes, we don't worry about consistency here.
         fun ideSummaryLine(key: String, nanos: Long) = String.format(
             // 13 is used for alignment here, because it's enough that 9.99sec will still
             // align with any other output, without moving data too far to the right
             "%13s ns %s",
             NumberFormat.getNumberInstance().format(nanos),
             key
         )

         fun Bundle.putIdeSummaryLine(testName: String, nanos: Long) {
             putString(
                 STUDIO_OUTPUT_KEY_PREFIX + STUDIO_OUTPUT_KEY_ID,
                 ideSummaryLineWrapped(testName, nanos)
             )
         }
     }
 }
	/*
	* Copyright (C) 2016 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	package androidx.benchmark

	import android.app.Activity
	import android.os.Build
	import android.os.Bundle
	import android.os.Debug
	import android.util.Log
	import androidx.annotation.IntRange
	import androidx.annotation.VisibleForTesting
	import androidx.test.platform.app.InstrumentationRegistry
	import java.io.File
	import java.text.NumberFormat
	import java.util.ArrayList
	import java.util.concurrent.TimeUnit

	/**
	* Control object for benchmarking in the code in Java.
	*
	* Query a state object with [BenchmarkRule.state], and use it to measure a block of Java with
	* [BenchmarkState.keepRunning]:
	* ```
	* @Rule
	* public BenchmarkRule benchmarkRule = new BenchmarkRule();
	*
	* @Test
	* public void sampleMethod() {
	* BenchmarkState state = benchmarkRule.getState();
	*
	* int[] src = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
	* while (state.keepRunning()) {
	* int[] dest = new int[src.length];
	* System.arraycopy(src, 0, dest, 0, src.length);
	* }
	* }
	* ```
	*
	* @see BenchmarkRule#getState()
	*/
	class BenchmarkState internal constructor() {
	private var warmupIteration = 0 // increasing iteration count during warmup

	/**
	* Decreasing iteration count used when [state] == [RUNNING], used to determine when main
	* measurement loop finishes.
	*/
	@JvmField // Used by [BenchmarkState.keepRunningInline()]
	@PublishedApi
	internal var iterationsRemaining = -1

	private var maxIterations = 0

	private var state = NOT_STARTED // Current benchmark state.

	private val warmupManager = WarmupManager()

	private var startTimeNs: Long = 0 // System.nanoTime() at start of last warmup/test iter.

	private var paused = false
	private var pausedTimeNs: Long = 0 // The System.nanoTime() when the pauseTiming() is called.
	private var pausedDurationNs: Long = 0 // The duration of paused state in nano sec.
	private var thermalThrottleSleepSeconds: Long =
	0 // The duration of sleep due to thermal throttling.

	private var repeatCount = 0

	// Statistics. These values will be filled when the benchmark has finished.
	// The computation needs double precision, but long int is fine for final reporting.
	private var internalStats: Stats? = null

	// Individual duration in nano seconds.
	private val results = ArrayList<Long>()

	internal var performThrottleChecks = true
	private var throttleRemainingRetries = THROTTLE_MAX_RETRIES

	/**
	* Get the end of run benchmark statistics.
	*
	*
	* This method may only be called keepRunning() returns `false`.
	*
	* @return Stats from run.
	*/
	internal val stats: Stats
	get() {
	if (state == NOT_STARTED) {
	throw IllegalStateException(
	"The benchmark wasn't started! Every test in a class " +
	"with a BenchmarkRule must contain a benchmark. In Kotlin, call " +
	"benchmarkRule.measureRepeated {}, or in Java, call " +
	"benchmarkRule.getState().keepRunning() to run your benchmark."
	)
	}
	if (state != FINISHED) {
	throw IllegalStateException(
	"The benchmark hasn't finished! In Java, use " +
	"while(BenchmarkState.keepRunning()) to ensure keepRunning() returns " +
	"false before ending your test. In Kotlin, just use " +
	"benchmarkRule.measureRepeated {} to avoid the problem."
	)
	}
	return internalStats!!
	}

	/**
	* Stops the benchmark timer.
	*
	* This method can be called only when the timer is running.
	*
	* ```
	* @Test
	* public void bitmapProcessing() {
	* final BenchmarkState state = mBenchmarkRule.getState();
	* while (state.keepRunning()) {
	* state.pauseTiming();
	* // disable timing while constructing test input
	* Bitmap input = constructTestBitmap();
	* state.resumeTiming();
	*
	* processBitmap(input);
	* }
	* }
	* ```
	*
	* @see resumeTiming
	*/
	fun pauseTiming() {
	if (paused) {
	throw IllegalStateException(
	"Unable to pause the benchmark. The benchmark has already paused."
	)
	}
	pausedTimeNs = System.nanoTime()
	paused = true
	}

	/**
	* Resumes the benchmark timer.
	*
	* This method can be called only when the timer is stopped.
	*
	* ```
	* @Test
	* public void bitmapProcessing() {
	* final BenchmarkState state = mBenchmarkRule.getState();
	* while (state.keepRunning()) {
	* state.pauseTiming();
	* // disable timing while constructing test input
	* Bitmap input = constructTestBitmap();
	* state.resumeTiming();
	*
	* processBitmap(input);
	* }
	* }
	* ```
	*
	* @see pauseTiming
	*/
	fun resumeTiming() {
	if (!paused) {
	throw IllegalStateException(
	"Unable to resume the benchmark. The benchmark is already running."
	)
	}
	pausedDurationNs += System.nanoTime() - pausedTimeNs
	pausedTimeNs = 0
	paused = false
	}

	private fun beginWarmup() {
	startTimeNs = System.nanoTime()
	warmupIteration = 0
	state = WARMUP
	}

	private fun beginBenchmark() {
	if (ENABLE_PROFILING && Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
	// TODO: support data dir for old platforms
	val f = File(
	InstrumentationRegistry.getInstrumentation().context.dataDir,
	"benchprof"
	)
	Log.d(TAG, "Tracing to: " + f.absolutePath)
	Debug.startMethodTracingSampling(f.absolutePath, 16 * 1024 * 1024, 100)
	}
	val idealIterations =
	(TARGET_TEST_DURATION_NS / warmupManager.estimatedIterationTime).toInt()
	maxIterations = Math.min(
	MAX_TEST_ITERATIONS,
	Math.max(idealIterations, MIN_TEST_ITERATIONS)
	)
	pausedDurationNs = 0
	iterationsRemaining = maxIterations
	repeatCount = 0
	thermalThrottleSleepSeconds = 0
	state = RUNNING
	startTimeNs = System.nanoTime()
	}

	private fun startNextTestRun(): Boolean {
	val currentTime = System.nanoTime()

	results.add((currentTime - startTimeNs - pausedDurationNs) / maxIterations)
	repeatCount++

	if (repeatCount >= REPEAT_COUNT) {
	if (performThrottleChecks &&
	throttleRemainingRetries > 0 &&
	sleepIfThermalThrottled(THROTTLE_BACKOFF_S)
	) {
	// We've slept due to thermal throttle - retry benchmark!
	throttleRemainingRetries -= 1
	results.clear()
	repeatCount = 0
	} else {
	// finished!
	if (ENABLE_PROFILING) {
	Debug.stopMethodTracing()
	}
	internalStats = Stats(results)
	state = FINISHED
	return false
	}
	}
	pausedDurationNs = 0
	iterationsRemaining = maxIterations
	startTimeNs = System.nanoTime()
	return true
	}

	/**
	* Inline fast-path function for inner benchmark loop.
	*
	* Kotlin users should use [BenchmarkRule.measureRepeated]
	*
	* This codepath uses exclusively @JvmField/const members, so there are no method calls at all
	* in the inlined loop. On recent Android Platform versions, ART inlines these accessors anyway,
	* but we want to be sure it's as simple as possible.
	*/
	@Suppress("NOTHING_TO_INLINE")
	@PublishedApi
	internal inline fun keepRunningInline(): Boolean {
	if (iterationsRemaining > 1) {
	iterationsRemaining--
	return true
	}
	return keepRunningInternal()
	}

	/**
	* Returns true if the benchmark needs more samples - use this as the condition of a while loop.
	*
	* ```
	* while (state.keepRunning()) {
	* int[] dest = new int[src.length];
	* System.arraycopy(src, 0, dest, 0, src.length);
	* }
	* ```
	*/
	fun keepRunning(): Boolean {
	if (iterationsRemaining > 1) {
	iterationsRemaining--
	return true
	}
	return keepRunningInternal()
	}

	@PublishedApi
	internal fun keepRunningInternal(): Boolean {
	when (state) {
	NOT_STARTED -> {
	if (performThrottleChecks &&
	!CpuInfo.locked &&
	!AndroidBenchmarkRunner.sustainedPerformanceModeInUse &&
	!WarningState.isEmulator
	) {
	ThrottleDetector.computeThrottleBaseline()
	}
	beginWarmup()
	return true
	}
	WARMUP -> {
	warmupIteration++
	// Only check nanoTime on every iteration in WARMUP since we
	// don't yet have a target iteration count.
	val time = System.nanoTime()
	val lastDuration = time - startTimeNs
	startTimeNs = time
	throwIfPaused() // check each loop during warmup
	if (warmupManager.onNextIteration(lastDuration)) {
	beginBenchmark()
	}
	return true
	}
	RUNNING -> {
	iterationsRemaining--
	if (iterationsRemaining <= 0) {
	throwIfPaused() // only check at end of loop to save cycles
	return startNextTestRun()
	}
	return true
	}
	FINISHED -> throw IllegalStateException("The benchmark has finished.")
	else -> throw IllegalStateException("The benchmark is in unknown state.")
	}
	}

	private fun throwIfPaused() {
	if (paused) {
	throw IllegalStateException(
	"Benchmark step finished with paused state." +
	" Resume the benchmark before finishing each step."
	)
	}
	}

	internal data class Report(
	val className: String,
	val testName: String,
	val data: List<Long>,
	val repeatIterations: Int,
	val thermalThrottleSleepSeconds: Long,
	val warmupIterations: Int
	) {
	val stats = Stats(data)
	}

	internal fun getReport(testName: String, className: String) = Report(
	className = className,
	testName = testName,
	data = results,
	repeatIterations = maxIterations,
	thermalThrottleSleepSeconds = thermalThrottleSleepSeconds,
	warmupIterations = warmupIteration
	)

	private fun summaryLine() = "Summary: " +
	"median=${stats.median}ns, " +
	"mean=${stats.mean.toLong()}ns, " +
	"min=${stats.min}ns, " +
	"stddev=${stats.standardDeviation.toLong()}ns, " +
	"count=$maxIterations"

	/**
	* Acquires a status report bundle
	*
	* @param key Run identifier, prepended to bundle properties.
	*/
	internal fun getFullStatusReport(key: String): Bundle {
	Log.i(TAG, key + summaryLine())
	val status = Bundle()

	val prefix = WarningState.WARNING_PREFIX
	status.putLong("${prefix}median", stats.median)
	status.putLong("${prefix}mean", stats.mean.toLong())
	status.putLong("${prefix}min", stats.min)
	status.putLong("${prefix}standardDeviation", stats.standardDeviation.toLong())
	status.putLong("${prefix}count", maxIterations.toLong())
	status.putIdeSummaryLine(key, stats.min)
	return status
	}

	internal fun sendStatus(testName: String) {
	val bundle = getFullStatusReport(testName)
	InstrumentationRegistry.getInstrumentation().sendStatus(Activity.RESULT_OK, bundle)
	}

	private fun sleepIfThermalThrottled(sleepSeconds: Long) = when {
	ThrottleDetector.isDeviceThermalThrottled() -> {
	Log.d(TAG, "THERMAL THROTTLE DETECTED, SLEEPING FOR $sleepSeconds SECONDS")
	val startTime = System.nanoTime()
	Thread.sleep(TimeUnit.SECONDS.toMillis(sleepSeconds))
	val sleepTime = System.nanoTime() - startTime
	thermalThrottleSleepSeconds += TimeUnit.NANOSECONDS.toSeconds(sleepTime)
	true
	}
	else -> false
	}

	internal companion object {
	private const val TAG = "Benchmark"
	private const val STUDIO_OUTPUT_KEY_PREFIX = "android.studio.display."
	private const val STUDIO_OUTPUT_KEY_ID = "benchmark"

	private const val ENABLE_PROFILING = false

	private const val NOT_STARTED = 0 // The benchmark has not started yet.
	private const val WARMUP = 1 // The benchmark is warming up.
	private const val RUNNING = 2 // The benchmark is running.
	private const val FINISHED = 3 // The benchmark has stopped.

	// Values determined empirically.
	@VisibleForTesting
	internal const val REPEAT_COUNT = 50
	private val TARGET_TEST_DURATION_NS = TimeUnit.MICROSECONDS.toNanos(500)
	private const val MAX_TEST_ITERATIONS = 1000000
	private const val MIN_TEST_ITERATIONS = 1

	private const val THROTTLE_MAX_RETRIES = 3
	private const val THROTTLE_BACKOFF_S = 90L

	/**
	* Hooks for benchmarks not using [BenchmarkRule] to register results.
	*
	* Results are printed to Studio console, and added to the output JSON file.
	*
	* @param className Name of class the benchmark runs in
	* @param testName Name of the benchmark
	* @param dataNs List of all measured results, in nanoseconds
	* @param warmupIterations Number of iterations of warmup before measurements started.
	* Should be no less than 0.
	* @param thermalThrottleSleepSeconds Number of seconds benchmark was paused during thermal
	* throttling.
	* @param repeatIterations Number of iterations in between each measurement. Should be no
	* less than 1.
	*/
	@Suppress("unused")
	@JvmStatic
	fun reportData(
	className: String,
	testName: String,
	dataNs: List<Long>,
	@IntRange(from = 0) warmupIterations: Int,
	@IntRange(from = 0) thermalThrottleSleepSeconds: Long,
	@IntRange(from = 1) repeatIterations: Int
	) {
	val report = Report(
	className = className,
	testName = testName,
	data = dataNs,
	repeatIterations = repeatIterations,
	thermalThrottleSleepSeconds = thermalThrottleSleepSeconds,
	warmupIterations = warmupIterations
	)

	// Report value to Studio console
	val bundle = Bundle()
	val fullTestName = WarningState.WARNING_PREFIX +
	if (className.isNotEmpty()) "$className.$testName" else testName
	bundle.putIdeSummaryLine(fullTestName, report.stats.min)
	InstrumentationRegistry.getInstrumentation().sendStatus(Activity.RESULT_OK, bundle)

	// Report values to file output
	ResultWriter.appendReport(report)
	}

	internal fun ideSummaryLineWrapped(key: String, nanos: Long): String {
	val warningLines =
	WarningState.acquireWarningStringForLogging()?.split("\n") ?: listOf()
	return (warningLines + ideSummaryLine(key, nanos))
	// remove first line if empty
	.filterIndexed { index, it -> index != 0 \|\| !it.isEmpty() }
	// join, prepending key to everything but first string,
	// to make each line look the same
	.joinToString("\n$STUDIO_OUTPUT_KEY_ID: ")
	}

	// NOTE: this summary line will use default locale to determine separators. As
	// this line is only meant for human eyes, we don't worry about consistency here.
	fun ideSummaryLine(key: String, nanos: Long) = String.format(
	// 13 is used for alignment here, because it's enough that 9.99sec will still
	// align with any other output, without moving data too far to the right
	"%13s ns %s",
	NumberFormat.getNumberInstance().format(nanos),
	key
	)

	fun Bundle.putIdeSummaryLine(testName: String, nanos: Long) {
	putString(
	STUDIO_OUTPUT_KEY_PREFIX + STUDIO_OUTPUT_KEY_ID,
	ideSummaryLineWrapped(testName, nanos)
	)
	}
	}
	}