[Chromecast] Add loopback input for AEC
Add i2s loopback in order to enable the reference channel for AEC.
Bug: http://b/38428792
Test: built cast_shell_internal_apk, ran assistant with AEC
Change-Id: Ie8c1ecf09bc31050e3cfe61eddf69dc21aba1fce
Reviewed-on: https://chromium-review.googlesource.com/566063
Commit-Queue: Nat Jeffries <njeff@google.com>
Reviewed-by: Luke Halliwell <halliwell@chromium.org>
Cr-Commit-Position: refs/heads/master@{#488801}
diff --git a/chromecast/BUILD.gn b/chromecast/BUILD.gn
index 27878e5a..0dbc08e 100644
--- a/chromecast/BUILD.gn
+++ b/chromecast/BUILD.gn
@@ -485,6 +485,7 @@
header = "chromecast_features.h"
flags = [
"ENABLE_ASSISTANT=$enable_assistant",
+ "ENABLE_ATHINGS_LOOPBACK=$enable_athings_loopback",
"IS_ANDROID_THINGS=$is_android_things",
"IS_CAST_AUDIO_ONLY=$is_cast_audio_only",
"IS_CAST_USING_CMA_BACKEND=$is_cast_using_cma_backend",
diff --git a/chromecast/base/chromecast_switches.cc b/chromecast/base/chromecast_switches.cc
index ce23b09..2ea2c4c 100644
--- a/chromecast/base/chromecast_switches.cc
+++ b/chromecast/base/chromecast_switches.cc
@@ -109,6 +109,12 @@
const char kCastInitialScreenHeight[] = "cast-initial-screen-height";
const char kUseDoubleBuffering[] = "use-double-buffering";
+// Used to pass configuration for the I2S input to enable loopback for AEC.
+const char kLoopbackI2sBits[] = "loopback-i2s-bits";
+const char kLoopbackI2sChannels[] = "loopback-i2s-channels";
+const char kLoopbackI2sNumber[] = "loopback-i2s-number";
+const char kLoopbackI2sRate[] = "loopback-i2s-rate";
+
// When present, desktop cast_shell will create 1080p window (provided display
// resolution is high enough). Otherwise, cast_shell defaults to 720p.
const char kDesktopWindow1080p[] = "desktop-window-1080p";
diff --git a/chromecast/base/chromecast_switches.h b/chromecast/base/chromecast_switches.h
index 298452e..0c327e5 100644
--- a/chromecast/base/chromecast_switches.h
+++ b/chromecast/base/chromecast_switches.h
@@ -60,6 +60,12 @@
extern const char kCastInitialScreenHeight[];
extern const char kUseDoubleBuffering[];
+// I2S loopback configuration switches
+extern const char kLoopbackI2sBits[];
+extern const char kLoopbackI2sChannels[];
+extern const char kLoopbackI2sNumber[];
+extern const char kLoopbackI2sRate[];
+
// Graphics switches
extern const char kDesktopWindow1080p[];
diff --git a/chromecast/chromecast.gni b/chromecast/chromecast.gni
index 8426bea..4310f91 100644
--- a/chromecast/chromecast.gni
+++ b/chromecast/chromecast.gni
@@ -50,6 +50,12 @@
}
declare_args() {
+ # Currently android things libraries live in internal. TODO(njeff): change
+ # this when Android Things API is moved to public
+ enable_athings_loopback = is_android_things && chromecast_branding != "public"
+}
+
+declare_args() {
# Use Playready CDMs for internal non-desktop builds.
use_playready = !is_cast_desktop_build && chromecast_branding != "public"
}
diff --git a/chromecast/media/cma/backend/android/BUILD.gn b/chromecast/media/cma/backend/android/BUILD.gn
index 17e82a72..60261ca 100644
--- a/chromecast/media/cma/backend/android/BUILD.gn
+++ b/chromecast/media/cma/backend/android/BUILD.gn
@@ -4,6 +4,7 @@
import("//build/config/android/config.gni")
import("//build/config/android/rules.gni")
+import("//chromecast/chromecast.gni")
source_set("cast_media_android") {
sources = [
@@ -34,6 +35,14 @@
"//chromecast/public/media",
"//media",
]
+
+ if (enable_athings_loopback) {
+ deps += [ "//chromecast/internal/android/prebuilt/things:support_lib" ]
+ sources += [
+ "loopback_audio_manager.cc",
+ "loopback_audio_manager.h",
+ ]
+ }
}
generate_jni("audio_track_jni_headers") {
diff --git a/chromecast/media/cma/backend/android/DEPS b/chromecast/media/cma/backend/android/DEPS
index 407d13c..427e38a 100644
--- a/chromecast/media/cma/backend/android/DEPS
+++ b/chromecast/media/cma/backend/android/DEPS
@@ -1,4 +1,5 @@
include_rules = [
"+jni",
"+media/filters",
+ "+chromecast/internal/android/prebuilt/things",
]
diff --git a/chromecast/media/cma/backend/android/cast_media_android.cc b/chromecast/media/cma/backend/android/cast_media_android.cc
index ff751f83..d4df095 100644
--- a/chromecast/media/cma/backend/android/cast_media_android.cc
+++ b/chromecast/media/cma/backend/android/cast_media_android.cc
@@ -2,8 +2,11 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
+
#include "base/at_exit.h"
#include "base/logging.h"
+#include "chromecast/chromecast_features.h"
+#include "chromecast/media/cma/backend/android/loopback_audio_manager.h"
#include "chromecast/media/cma/backend/android/media_pipeline_backend_android.h"
#include "chromecast/public/cast_media_shlib.h"
#include "chromecast/public/graphics_types.h"
@@ -90,16 +93,16 @@
return false;
}
-#if 0 // disable for now, since libassistant doesn't handle a stub well.
+#if BUILDFLAG(ENABLE_ATHINGS_LOOPBACK)
void CastMediaShlib::AddLoopbackAudioObserver(LoopbackAudioObserver* observer) {
LOG(INFO) << __func__ << ":";
- // TODO(ckuiper): Hook-up if applicable.
+ LoopbackAudioManager::Get()->AddLoopbackAudioObserver(observer);
}
void CastMediaShlib::RemoveLoopbackAudioObserver(
LoopbackAudioObserver* observer) {
LOG(INFO) << __func__ << ":";
- // TODO(ckuiper): Hook-up if applicable.
+ LoopbackAudioManager::Get()->RemoveLoopbackAudioObserver(observer);
}
#endif
diff --git a/chromecast/media/cma/backend/android/loopback_audio_manager.cc b/chromecast/media/cma/backend/android/loopback_audio_manager.cc
new file mode 100644
index 0000000..68aa0d7
--- /dev/null
+++ b/chromecast/media/cma/backend/android/loopback_audio_manager.cc
@@ -0,0 +1,259 @@
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chromecast/media/cma/backend/android/loopback_audio_manager.h"
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include <string>
+
+#include "base/bind.h"
+#include "base/command_line.h"
+#include "base/lazy_instance.h"
+#include "base/logging.h"
+#include "chromecast/base/chromecast_switches.h"
+#include "chromecast/base/task_runner_impl.h"
+#include "chromecast/internal/android/prebuilt/things/include/pio/i2s_device.h"
+#include "chromecast/internal/android/prebuilt/things/include/pio/peripheral_manager_client.h"
+#include "chromecast/public/cast_media_shlib.h"
+#include "chromecast/public/media/decoder_config.h"
+
+#define RUN_ON_FEEDER_THREAD(method, ...) \
+ if (!feeder_thread_.task_runner()->BelongsToCurrentThread()) { \
+ POST_TASK_TO_FEEDER_THREAD(method, ##__VA_ARGS__); \
+ return; \
+ }
+
+#define POST_TASK_TO_FEEDER_THREAD(method, ...) \
+ feeder_thread_.task_runner()->PostTask( \
+ FROM_HERE, base::BindOnce(&LoopbackAudioManager::method, \
+ base::Unretained(this), ##__VA_ARGS__));
+
+namespace chromecast {
+namespace media {
+
+namespace {
+
+const int kMaxI2sNameLen = 32;
+const int kBufferLenMs = 20;
+const int kMsPerSecond 1000;
+const int64_t kNsecPerSecond = 1000000000LL;
+const int64_t kTimestampUpdatePeriodNsec = 10 * kNsecPerSecond;
+const int64_t kInvalidTimestamp = std::numeric_limits<int64_t>::min();
+
+class LoopbackAudioManagerInstance : public LoopbackAudioManager {
+ public:
+ LoopbackAudioManagerInstance() {}
+ ~LoopbackAudioManagerInstance() override {}
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(LoopbackAudioManagerInstance);
+};
+
+base::LazyInstance<LoopbackAudioManagerInstance>::DestructorAtExit
+ loopback_audio_manager_instance = LAZY_INSTANCE_INITIALIZER;
+
+} // namespace
+
+LoopbackAudioManager::LoopbackAudioManager()
+ : feeder_thread_("Android_Loopback"),
+ last_timestamp_nsec_(kInvalidTimestamp),
+ last_frame_position_(0),
+ frame_count_(0),
+ loopback_running(false) {
+ base::Thread::Options options;
+ options.priority = base::ThreadPriority::REALTIME_AUDIO;
+ CHECK(feeder_thread_.StartWithOptions(options));
+}
+
+LoopbackAudioManager::~LoopbackAudioManager() {
+ DCHECK(loopback_observers_.empty());
+ feeder_thread_.Stop();
+
+ // thread_.Stop() makes sure the thread is stopped before return.
+ // It's okay to clean up after feeder_thread_ is stopped.
+ StopLoopback();
+}
+
+// static
+LoopbackAudioManager* LoopbackAudioManager::Get() {
+ return loopback_audio_manager_instance.Pointer();
+}
+
+void LoopbackAudioManager::CalibrateTimestamp(int64_t frame_position) {
+ // Determine if a new calibration timestamp is needed.
+ struct timespec ts;
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ int64_t time_sec = static_cast<int64_t>(ts.tv_sec);
+ int64_t time_nsec = static_cast<int64_t>(ts.tv_nsec);
+ time_nsec += time_sec * kNsecPerSecond;
+ if (last_timestamp_nsec_ != kInvalidTimestamp &&
+ time_nsec - last_timestamp_nsec_ < kTimestampUpdatePeriodNsec) {
+ return;
+ }
+
+ // Validate that timestamp is close to interpolated estimate.
+ int64_t old_last_position = last_frame_position_;
+ int64_t old_last_timestamp = last_timestamp_nsec_;
+ int success;
+ DCHECK_EQ(0, AI2sDevice_getInputTimestamp(i2s_, &last_frame_position_,
+ &last_timestamp_nsec_, &success));
+ // If the call fails, the values are not updated.
+ if (!success) {
+ return;
+ }
+ int64_t delta_frames = last_frame_position_ - old_last_position;
+ int64_t delta_nsecs = kNsecPerSecond * delta_frames / i2s_rate_;
+ int64_t expected_timestamp = old_last_timestamp + delta_nsecs;
+ int64_t ts_diff_micros = (last_timestamp_nsec_ - expected_timestamp) / 1000;
+ if (ts_diff_micros > 1000) {
+ LOG(WARNING) << "Updated timestamp, interpolated timestamp drifted by "
+ << ts_diff_micros << " microseconds";
+ }
+}
+
+int64_t LoopbackAudioManager::GetInterpolatedTimestamp(int64_t frame_position) {
+ CalibrateTimestamp(frame_position);
+ int64_t delta_frames =
+ frame_position - last_frame_position_ -
+ audio_buffer_size_ / (bytes_per_sample_ * i2s_channels_);
+ int64_t delta_nsecs = kNsecPerSecond * delta_frames / i2s_rate_;
+ return last_timestamp_nsec_ + delta_nsecs;
+}
+
+void LoopbackAudioManager::GetI2sFlags(char* i2s_name,
+ AI2sEncoding* i2s_encoding) {
+ int i2s_number = GetSwitchValueInt(switches::kLoopbackI2sNumber, -1);
+ LOG_IF(DFATAL, i2s_number == -1)
+ << "Flag --" << switches::kLoopbackI2sNumber << " is required.";
+
+ sprintf(i2s_name, "I2S%d", i2s_number);
+ i2s_rate_ = GetSwitchValueNonNegativeInt(switches::kLoopbackI2sRate, 0);
+ LOG_IF(DFATAL, !i2s_rate_)
+ << "Flag --" << switches::kLoopbackI2sRate << " is required.";
+
+ int i2s_bits = GetSwitchValueNonNegativeInt(switches::kLoopbackI2sBits, 0);
+ LOG_IF(DFATAL, !i2s_bits)
+ << "Flag --" << switches::kLoopbackI2sBits << " is required.";
+ switch (i2s_bits) {
+ case 16:
+ *i2s_encoding = AI2S_ENCODING_PCM_16_BIT;
+ cast_audio_format_ = kSampleFormatS16;
+ bytes_per_sample_ = 2;
+ break;
+ case 24:
+ *i2s_encoding = AI2S_ENCODING_PCM_24_BIT;
+ cast_audio_format_ = kSampleFormatS24;
+ bytes_per_sample_ = 4; // 24-bits algined to 32 bits
+ break;
+ case 32:
+ *i2s_encoding = AI2S_ENCODING_PCM_32_BIT;
+ cast_audio_format_ = kSampleFormatS32;
+ bytes_per_sample_ = 4;
+ break;
+ default:
+ LOG(FATAL)
+ << "Invalid number of bits specified. Must be one of {16, 24, 32}";
+ }
+
+ i2s_channels_ =
+ GetSwitchValueNonNegativeInt(switches::kLoopbackI2sChannels, 0);
+ LOG_IF(DFATAL, !i2s_channels_)
+ << "Flag --" << switches::kLoopbackI2sChannels << " is required.";
+
+ // 20ms of audio
+ audio_buffer_size_ =
+ i2s_rate_ * bytes_per_sample_ * i2s_channels_ * kBufferLenMs / kMsPerSecond;
+}
+
+void LoopbackAudioManager::StartLoopback() {
+ DCHECK(feeder_thread_.task_runner()->BelongsToCurrentThread());
+ DCHECK(!loopback_running_);
+ last_timestamp_nsec_ = kInvalidTimestamp;
+
+ // Open I2S device.
+ APeripheralManagerClient* client = APeripheralManagerClient_new();
+ DCHECK(client);
+
+ char i2s_name[kMaxI2sNameLen];
+ AI2sEncoding i2s_encoding;
+ GetI2sFlags(i2s_name, &i2s_encoding);
+
+ int err = APeripheralManagerClient_openI2sDevice(
+ client, i2s_name, i2s_encoding, i2s_channels_, i2s_rate_,
+ AI2S_FLAG_DIRECTION_IN, &i2s_);
+ DCHECK_EQ(err, 0);
+
+ // Maintain sample count for interpolation.
+ frame_count_ = 0;
+ loopback_running_ = true;
+ RunLoopback();
+}
+
+void LoopbackAudioManager::StopLoopback() {
+ DCHECK(loopback_running_);
+ loopback_running_ = false;
+ AI2sDevice_delete(i2s_);
+}
+
+void LoopbackAudioManager::RunLoopback() {
+ DCHECK(feeder_thread_.task_runner()->BelongsToCurrentThread());
+
+ if (!loopback_running_) {
+ return;
+ }
+
+ // Read bytes from I2S device.
+ int bytes_read;
+ uint8_t data[audio_buffer_size_];
+ AI2sDevice_read(i2s_, data, 0, audio_buffer_size_, &bytes_read);
+ DCHECK_EQ(audio_buffer_size_, bytes_read);
+ frame_count_ += bytes_read / (bytes_per_sample_ * i2s_channels_);
+
+ // Get high-resolution timestamp.
+ int64_t timestamp_ns = GetInterpolatedTimestamp(frame_count_);
+
+ // Post data and timestamp.
+ for (auto* observer : loopback_observers_) {
+ observer->OnLoopbackAudio(timestamp_ns / 1000, cast_audio_format_,
+ i2s_rate_, i2s_channels_, data,
+ audio_buffer_size_);
+ }
+
+ POST_TASK_TO_FEEDER_THREAD(RunLoopback);
+}
+
+void LoopbackAudioManager::AddLoopbackAudioObserver(
+ CastMediaShlib::LoopbackAudioObserver* observer) {
+ DCHECK(observer);
+ RUN_ON_FEEDER_THREAD(AddLoopbackAudioObserver, observer);
+
+ DCHECK(std::find(loopback_observers_.begin(), loopback_observers_.end(),
+ observer) == loopback_observers_.end());
+ loopback_observers_.push_back(observer);
+ if (loopback_observers_.size() == 1) {
+ StartLoopback();
+ }
+}
+
+void LoopbackAudioManager::RemoveLoopbackAudioObserver(
+ CastMediaShlib::LoopbackAudioObserver* observer) {
+ DCHECK(observer);
+ RUN_ON_FEEDER_THREAD(RemoveLoopbackAudioObserver, observer);
+
+ DCHECK(std::find(loopback_observers_.begin(), loopback_observers_.end(),
+ observer) != loopback_observers_.end());
+ loopback_observers_.erase(std::remove(loopback_observers_.begin(),
+ loopback_observers_.end(), observer),
+ loopback_observers_.end());
+ observer->OnRemoved();
+ if (loopback_observers_.empty()) {
+ StopLoopback();
+ }
+}
+
+} // namespace media
+} // namespace chromecast
diff --git a/chromecast/media/cma/backend/android/loopback_audio_manager.h b/chromecast/media/cma/backend/android/loopback_audio_manager.h
new file mode 100644
index 0000000..ed68b4b
--- /dev/null
+++ b/chromecast/media/cma/backend/android/loopback_audio_manager.h
@@ -0,0 +1,73 @@
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMECAST_MEDIA_CMA_BACKEND_ANDROID_LOOPBACK_AUDIO_MANAGER_H_
+#define CHROMECAST_MEDIA_CMA_BACKEND_ANDROID_LOOPBACK_AUDIO_MANAGER_H_
+
+#include <stdint.h>
+
+#include <vector>
+
+#include "base/macros.h"
+#include "base/threading/thread.h"
+#include "chromecast/internal/android/prebuilt/things/include/pio/i2s_device.h"
+#include "chromecast/public/cast_media_shlib.h"
+
+struct AI2sDevice;
+
+namespace chromecast {
+namespace media {
+
+class LoopbackAudioManager {
+ public:
+ // Get singleton instance of Loopback Audio Manager
+ static LoopbackAudioManager* Get();
+
+ // Adds a loopback audio observer.
+ void AddLoopbackAudioObserver(
+ CastMediaShlib::LoopbackAudioObserver* observer);
+
+ // Removes a loopback audio observer.
+ void RemoveLoopbackAudioObserver(
+ CastMediaShlib::LoopbackAudioObserver* observer);
+
+ protected:
+ LoopbackAudioManager();
+ virtual ~LoopbackAudioManager();
+
+ private:
+ void StartLoopback();
+ void StopLoopback();
+ void RunLoopback();
+ void CalibrateTimestamp(int64_t frame_position);
+ int64_t GetInterpolatedTimestamp(int64_t frame_position);
+ void GetI2sFlags(char* i2s_name, AI2sEncoding* i2s_encoding);
+
+ std::vector<CastMediaShlib::LoopbackAudioObserver*> loopback_observers_;
+
+ AI2sDevice* i2s_;
+
+ int64_t last_timestamp_nsec_;
+ int64_t last_frame_position_;
+ int64_t frame_count_;
+
+ // Initialized based on the values of castshell flags
+ int audio_buffer_size_;
+ int bytes_per_sample_;
+ int i2s_rate_;
+ int i2s_channels_;
+ SampleFormat cast_audio_format_;
+
+ // Thread that feeds audio data into the observer from the loopback. The
+ // calls made on this thread are blocking.
+ base::Thread feeder_thread_;
+ bool loopback_running_;
+
+ DISALLOW_COPY_AND_ASSIGN(LoopbackAudioManager);
+};
+
+} // namespace media
+} // namespace chromecast
+
+#endif // CHROMECAST_MEDIA_CMA_BACKEND_ANDROID_LOOPBACK_AUDIO_MANAGER_H_