[go: nahoru, domu]
1/*
2 * Copyright (C) 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include <inttypes.h>
18
19#define LOG_TAG "GraphicBufferSource"
20//#define LOG_NDEBUG 0
21#include <utils/Log.h>
22
23#define STRINGIFY_ENUMS // for asString in HardwareAPI.h/VideoAPI.h
24
25#include "GraphicBufferSource.h"
26#include "OMXUtils.h"
27
28#include <OMX_Core.h>
29#include <OMX_IndexExt.h>
30#include <media/stagefright/foundation/ADebug.h>
31#include <media/stagefright/foundation/AMessage.h>
32#include <media/stagefright/foundation/ColorUtils.h>
33
34#include <media/hardware/MetadataBufferType.h>
35#include <ui/GraphicBuffer.h>
36#include <gui/BufferItem.h>
37#include <HardwareAPI.h>
38
39#include <inttypes.h>
40#include "FrameDropper.h"
41
42namespace android {
43
44static const bool EXTRA_CHECK = true;
45
46static const OMX_U32 kPortIndexInput = 0;
47
48GraphicBufferSource::PersistentProxyListener::PersistentProxyListener(
49        const wp<IGraphicBufferConsumer> &consumer,
50        const wp<ConsumerListener>& consumerListener) :
51    mConsumerListener(consumerListener),
52    mConsumer(consumer) {}
53
54GraphicBufferSource::PersistentProxyListener::~PersistentProxyListener() {}
55
56void GraphicBufferSource::PersistentProxyListener::onFrameAvailable(
57        const BufferItem& item) {
58    sp<ConsumerListener> listener(mConsumerListener.promote());
59    if (listener != NULL) {
60        listener->onFrameAvailable(item);
61    } else {
62        sp<IGraphicBufferConsumer> consumer(mConsumer.promote());
63        if (consumer == NULL) {
64            return;
65        }
66        BufferItem bi;
67        status_t err = consumer->acquireBuffer(&bi, 0);
68        if (err != OK) {
69            ALOGE("PersistentProxyListener: acquireBuffer failed (%d)", err);
70            return;
71        }
72
73        err = consumer->detachBuffer(bi.mSlot);
74        if (err != OK) {
75            ALOGE("PersistentProxyListener: detachBuffer failed (%d)", err);
76            return;
77        }
78
79        err = consumer->attachBuffer(&bi.mSlot, bi.mGraphicBuffer);
80        if (err != OK) {
81            ALOGE("PersistentProxyListener: attachBuffer failed (%d)", err);
82            return;
83        }
84
85        err = consumer->releaseBuffer(bi.mSlot, 0,
86                EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, bi.mFence);
87        if (err != OK) {
88            ALOGE("PersistentProxyListener: releaseBuffer failed (%d)", err);
89        }
90    }
91}
92
93void GraphicBufferSource::PersistentProxyListener::onFrameReplaced(
94        const BufferItem& item) {
95    sp<ConsumerListener> listener(mConsumerListener.promote());
96    if (listener != NULL) {
97        listener->onFrameReplaced(item);
98    }
99}
100
101void GraphicBufferSource::PersistentProxyListener::onBuffersReleased() {
102    sp<ConsumerListener> listener(mConsumerListener.promote());
103    if (listener != NULL) {
104        listener->onBuffersReleased();
105    }
106}
107
108void GraphicBufferSource::PersistentProxyListener::onSidebandStreamChanged() {
109    sp<ConsumerListener> listener(mConsumerListener.promote());
110    if (listener != NULL) {
111        listener->onSidebandStreamChanged();
112    }
113}
114
115GraphicBufferSource::GraphicBufferSource(
116        OMXNodeInstance* nodeInstance,
117        uint32_t bufferWidth,
118        uint32_t bufferHeight,
119        uint32_t bufferCount,
120        uint32_t consumerUsage,
121        const sp<IGraphicBufferConsumer> &consumer) :
122    mInitCheck(UNKNOWN_ERROR),
123    mNodeInstance(nodeInstance),
124    mExecuting(false),
125    mSuspended(false),
126    mLastDataSpace(HAL_DATASPACE_UNKNOWN),
127    mIsPersistent(false),
128    mConsumer(consumer),
129    mNumFramesAvailable(0),
130    mNumBufferAcquired(0),
131    mEndOfStream(false),
132    mEndOfStreamSent(false),
133    mMaxTimestampGapUs(-1ll),
134    mPrevOriginalTimeUs(-1ll),
135    mPrevModifiedTimeUs(-1ll),
136    mSkipFramesBeforeNs(-1ll),
137    mRepeatAfterUs(-1ll),
138    mRepeatLastFrameGeneration(0),
139    mRepeatLastFrameTimestamp(-1ll),
140    mLatestBufferId(-1),
141    mLatestBufferFrameNum(0),
142    mLatestBufferUseCount(0),
143    mLatestBufferFence(Fence::NO_FENCE),
144    mRepeatBufferDeferred(false),
145    mTimePerCaptureUs(-1ll),
146    mTimePerFrameUs(-1ll),
147    mPrevCaptureUs(-1ll),
148    mPrevFrameUs(-1ll) {
149
150    ALOGV("GraphicBufferSource w=%u h=%u c=%u",
151            bufferWidth, bufferHeight, bufferCount);
152
153    if (bufferWidth == 0 || bufferHeight == 0) {
154        ALOGE("Invalid dimensions %ux%u", bufferWidth, bufferHeight);
155        mInitCheck = BAD_VALUE;
156        return;
157    }
158
159    if (mConsumer == NULL) {
160        String8 name("GraphicBufferSource");
161
162        BufferQueue::createBufferQueue(&mProducer, &mConsumer);
163        mConsumer->setConsumerName(name);
164
165        // use consumer usage bits queried from encoder, but always add HW_VIDEO_ENCODER
166        // for backward compatibility.
167        consumerUsage |= GRALLOC_USAGE_HW_VIDEO_ENCODER;
168        mConsumer->setConsumerUsageBits(consumerUsage);
169
170        mInitCheck = mConsumer->setMaxAcquiredBufferCount(bufferCount);
171        if (mInitCheck != NO_ERROR) {
172            ALOGE("Unable to set BQ max acquired buffer count to %u: %d",
173                    bufferCount, mInitCheck);
174            return;
175        }
176    } else {
177        mIsPersistent = true;
178    }
179    mConsumer->setDefaultBufferSize(bufferWidth, bufferHeight);
180    // Note that we can't create an sp<...>(this) in a ctor that will not keep a
181    // reference once the ctor ends, as that would cause the refcount of 'this'
182    // dropping to 0 at the end of the ctor.  Since all we need is a wp<...>
183    // that's what we create.
184    wp<BufferQueue::ConsumerListener> listener = static_cast<BufferQueue::ConsumerListener*>(this);
185    sp<IConsumerListener> proxy;
186    if (!mIsPersistent) {
187        proxy = new BufferQueue::ProxyConsumerListener(listener);
188    } else {
189        proxy = new PersistentProxyListener(mConsumer, listener);
190    }
191
192    mInitCheck = mConsumer->consumerConnect(proxy, false);
193    if (mInitCheck != NO_ERROR) {
194        ALOGE("Error connecting to BufferQueue: %s (%d)",
195                strerror(-mInitCheck), mInitCheck);
196        return;
197    }
198
199    memset(&mColorAspects, 0, sizeof(mColorAspects));
200
201    CHECK(mInitCheck == NO_ERROR);
202}
203
204GraphicBufferSource::~GraphicBufferSource() {
205    if (mLatestBufferId >= 0) {
206        releaseBuffer(
207                mLatestBufferId, mLatestBufferFrameNum,
208                mBufferSlot[mLatestBufferId], mLatestBufferFence);
209    }
210    if (mNumBufferAcquired != 0) {
211        ALOGW("potential buffer leak (acquired %d)", mNumBufferAcquired);
212    }
213    if (mConsumer != NULL && !mIsPersistent) {
214        status_t err = mConsumer->consumerDisconnect();
215        if (err != NO_ERROR) {
216            ALOGW("consumerDisconnect failed: %d", err);
217        }
218    }
219}
220
221void GraphicBufferSource::omxExecuting() {
222    Mutex::Autolock autoLock(mMutex);
223    ALOGV("--> executing; avail=%zu, codec vec size=%zd",
224            mNumFramesAvailable, mCodecBuffers.size());
225    CHECK(!mExecuting);
226    mExecuting = true;
227    mLastDataSpace = HAL_DATASPACE_UNKNOWN;
228    ALOGV("clearing last dataSpace");
229
230    // Start by loading up as many buffers as possible.  We want to do this,
231    // rather than just submit the first buffer, to avoid a degenerate case:
232    // if all BQ buffers arrive before we start executing, and we only submit
233    // one here, the other BQ buffers will just sit until we get notified
234    // that the codec buffer has been released.  We'd then acquire and
235    // submit a single additional buffer, repeatedly, never using more than
236    // one codec buffer simultaneously.  (We could instead try to submit
237    // all BQ buffers whenever any codec buffer is freed, but if we get the
238    // initial conditions right that will never be useful.)
239    while (mNumFramesAvailable) {
240        if (!fillCodecBuffer_l()) {
241            ALOGV("stop load with frames available (codecAvail=%d)",
242                    isCodecBufferAvailable_l());
243            break;
244        }
245    }
246
247    ALOGV("done loading initial frames, avail=%zu", mNumFramesAvailable);
248
249    // If EOS has already been signaled, and there are no more frames to
250    // submit, try to send EOS now as well.
251    if (mEndOfStream && mNumFramesAvailable == 0) {
252        submitEndOfInputStream_l();
253    }
254
255    if (mRepeatAfterUs > 0ll && mLooper == NULL) {
256        mReflector = new AHandlerReflector<GraphicBufferSource>(this);
257
258        mLooper = new ALooper;
259        mLooper->registerHandler(mReflector);
260        mLooper->start();
261
262        if (mLatestBufferId >= 0) {
263            sp<AMessage> msg =
264                new AMessage(kWhatRepeatLastFrame, mReflector);
265
266            msg->setInt32("generation", ++mRepeatLastFrameGeneration);
267            msg->post(mRepeatAfterUs);
268        }
269    }
270}
271
272void GraphicBufferSource::omxIdle() {
273    ALOGV("omxIdle");
274
275    Mutex::Autolock autoLock(mMutex);
276
277    if (mExecuting) {
278        // We are only interested in the transition from executing->idle,
279        // not loaded->idle.
280        mExecuting = false;
281    }
282}
283
284void GraphicBufferSource::omxLoaded(){
285    Mutex::Autolock autoLock(mMutex);
286    if (!mExecuting) {
287        // This can happen if something failed very early.
288        ALOGW("Dropped back down to Loaded without Executing");
289    }
290
291    if (mLooper != NULL) {
292        mLooper->unregisterHandler(mReflector->id());
293        mReflector.clear();
294
295        mLooper->stop();
296        mLooper.clear();
297    }
298
299    ALOGV("--> loaded; avail=%zu eos=%d eosSent=%d",
300            mNumFramesAvailable, mEndOfStream, mEndOfStreamSent);
301
302    // Codec is no longer executing.  Discard all codec-related state.
303    mCodecBuffers.clear();
304    // TODO: scan mCodecBuffers to verify that all mGraphicBuffer entries
305    //       are null; complain if not
306
307    mExecuting = false;
308}
309
310void GraphicBufferSource::addCodecBuffer(OMX_BUFFERHEADERTYPE* header) {
311    Mutex::Autolock autoLock(mMutex);
312
313    if (mExecuting) {
314        // This should never happen -- buffers can only be allocated when
315        // transitioning from "loaded" to "idle".
316        ALOGE("addCodecBuffer: buffer added while executing");
317        return;
318    }
319
320    ALOGV("addCodecBuffer h=%p size=%" PRIu32 " p=%p",
321            header, header->nAllocLen, header->pBuffer);
322    CodecBuffer codecBuffer;
323    codecBuffer.mHeader = header;
324    mCodecBuffers.add(codecBuffer);
325}
326
327void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header, int fenceFd) {
328    Mutex::Autolock autoLock(mMutex);
329    if (!mExecuting) {
330        return;
331    }
332
333    int cbi = findMatchingCodecBuffer_l(header);
334    if (cbi < 0) {
335        // This should never happen.
336        ALOGE("codecBufferEmptied: buffer not recognized (h=%p)", header);
337        if (fenceFd >= 0) {
338            ::close(fenceFd);
339        }
340        return;
341    }
342
343    ALOGV("codecBufferEmptied h=%p size=%" PRIu32 " filled=%" PRIu32 " p=%p",
344            header, header->nAllocLen, header->nFilledLen,
345            header->pBuffer);
346    CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
347
348    // header->nFilledLen may not be the original value, so we can't compare
349    // that to zero to see of this was the EOS buffer.  Instead we just
350    // see if the GraphicBuffer reference was null, which should only ever
351    // happen for EOS.
352    if (codecBuffer.mGraphicBuffer == NULL) {
353        if (!(mEndOfStream && mEndOfStreamSent)) {
354            // This can happen when broken code sends us the same buffer
355            // twice in a row.
356            ALOGE("ERROR: codecBufferEmptied on non-EOS null buffer "
357                    "(buffer emptied twice?)");
358        }
359        // No GraphicBuffer to deal with, no additional input or output is
360        // expected, so just return.
361        if (fenceFd >= 0) {
362            ::close(fenceFd);
363        }
364        return;
365    }
366
367    if (EXTRA_CHECK && header->nAllocLen >= sizeof(MetadataBufferType)) {
368        // Pull the graphic buffer handle back out of the buffer, and confirm
369        // that it matches expectations.
370        OMX_U8* data = header->pBuffer;
371        MetadataBufferType type = *(MetadataBufferType *)data;
372        if (type == kMetadataBufferTypeGrallocSource
373                && header->nAllocLen >= sizeof(VideoGrallocMetadata)) {
374            VideoGrallocMetadata &grallocMeta = *(VideoGrallocMetadata *)data;
375            if (grallocMeta.pHandle != codecBuffer.mGraphicBuffer->handle) {
376                // should never happen
377                ALOGE("codecBufferEmptied: buffer's handle is %p, expected %p",
378                        grallocMeta.pHandle, codecBuffer.mGraphicBuffer->handle);
379                CHECK(!"codecBufferEmptied: mismatched buffer");
380            }
381        } else if (type == kMetadataBufferTypeANWBuffer
382                && header->nAllocLen >= sizeof(VideoNativeMetadata)) {
383            VideoNativeMetadata &nativeMeta = *(VideoNativeMetadata *)data;
384            if (nativeMeta.pBuffer != codecBuffer.mGraphicBuffer->getNativeBuffer()) {
385                // should never happen
386                ALOGE("codecBufferEmptied: buffer is %p, expected %p",
387                        nativeMeta.pBuffer, codecBuffer.mGraphicBuffer->getNativeBuffer());
388                CHECK(!"codecBufferEmptied: mismatched buffer");
389            }
390        }
391    }
392
393    // Find matching entry in our cached copy of the BufferQueue slots.
394    // If we find a match, release that slot.  If we don't, the BufferQueue
395    // has dropped that GraphicBuffer, and there's nothing for us to release.
396    int id = codecBuffer.mSlot;
397    sp<Fence> fence = new Fence(fenceFd);
398    if (mBufferSlot[id] != NULL &&
399        mBufferSlot[id]->handle == codecBuffer.mGraphicBuffer->handle) {
400        ALOGV("cbi %d matches bq slot %d, handle=%p",
401                cbi, id, mBufferSlot[id]->handle);
402
403        if (id == mLatestBufferId) {
404            CHECK_GT(mLatestBufferUseCount--, 0);
405        } else {
406            releaseBuffer(id, codecBuffer.mFrameNumber, mBufferSlot[id], fence);
407        }
408    } else {
409        ALOGV("codecBufferEmptied: no match for emptied buffer in cbi %d",
410                cbi);
411        // we will not reuse codec buffer, so there is no need to wait for fence
412    }
413
414    // Mark the codec buffer as available by clearing the GraphicBuffer ref.
415    codecBuffer.mGraphicBuffer = NULL;
416
417    if (mNumFramesAvailable) {
418        // Fill this codec buffer.
419        CHECK(!mEndOfStreamSent);
420        ALOGV("buffer freed, %zu frames avail (eos=%d)",
421                mNumFramesAvailable, mEndOfStream);
422        fillCodecBuffer_l();
423    } else if (mEndOfStream) {
424        // No frames available, but EOS is pending, so use this buffer to
425        // send that.
426        ALOGV("buffer freed, EOS pending");
427        submitEndOfInputStream_l();
428    } else if (mRepeatBufferDeferred) {
429        bool success = repeatLatestBuffer_l();
430        if (success) {
431            ALOGV("deferred repeatLatestBuffer_l SUCCESS");
432        } else {
433            ALOGV("deferred repeatLatestBuffer_l FAILURE");
434        }
435        mRepeatBufferDeferred = false;
436    }
437
438    return;
439}
440
441void GraphicBufferSource::codecBufferFilled(OMX_BUFFERHEADERTYPE* header) {
442    Mutex::Autolock autoLock(mMutex);
443
444    if (mMaxTimestampGapUs > 0ll
445            && !(header->nFlags & OMX_BUFFERFLAG_CODECCONFIG)) {
446        ssize_t index = mOriginalTimeUs.indexOfKey(header->nTimeStamp);
447        if (index >= 0) {
448            ALOGV("OUT timestamp: %lld -> %lld",
449                    static_cast<long long>(header->nTimeStamp),
450                    static_cast<long long>(mOriginalTimeUs[index]));
451            header->nTimeStamp = mOriginalTimeUs[index];
452            mOriginalTimeUs.removeItemsAt(index);
453        } else {
454            // giving up the effort as encoder doesn't appear to preserve pts
455            ALOGW("giving up limiting timestamp gap (pts = %lld)",
456                    header->nTimeStamp);
457            mMaxTimestampGapUs = -1ll;
458        }
459        if (mOriginalTimeUs.size() > BufferQueue::NUM_BUFFER_SLOTS) {
460            // something terribly wrong must have happened, giving up...
461            ALOGE("mOriginalTimeUs has too many entries (%zu)",
462                    mOriginalTimeUs.size());
463            mMaxTimestampGapUs = -1ll;
464        }
465    }
466}
467
468void GraphicBufferSource::suspend(bool suspend) {
469    Mutex::Autolock autoLock(mMutex);
470
471    if (suspend) {
472        mSuspended = true;
473
474        while (mNumFramesAvailable > 0) {
475            BufferItem item;
476            status_t err = mConsumer->acquireBuffer(&item, 0);
477
478            if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
479                // shouldn't happen.
480                ALOGW("suspend: frame was not available");
481                break;
482            } else if (err != OK) {
483                ALOGW("suspend: acquireBuffer returned err=%d", err);
484                break;
485            }
486
487            ++mNumBufferAcquired;
488            --mNumFramesAvailable;
489
490            releaseBuffer(item.mSlot, item.mFrameNumber,
491                    item.mGraphicBuffer, item.mFence);
492        }
493        return;
494    }
495
496    mSuspended = false;
497
498    if (mExecuting && mNumFramesAvailable == 0 && mRepeatBufferDeferred) {
499        if (repeatLatestBuffer_l()) {
500            ALOGV("suspend/deferred repeatLatestBuffer_l SUCCESS");
501
502            mRepeatBufferDeferred = false;
503        } else {
504            ALOGV("suspend/deferred repeatLatestBuffer_l FAILURE");
505        }
506    }
507}
508
509void GraphicBufferSource::onDataSpaceChanged_l(
510        android_dataspace dataSpace, android_pixel_format pixelFormat) {
511    ALOGD("got buffer with new dataSpace #%x", dataSpace);
512    mLastDataSpace = dataSpace;
513
514    if (ColorUtils::convertDataSpaceToV0(dataSpace)) {
515        ColorAspects aspects = mColorAspects; // initially requested aspects
516
517        // request color aspects to encode
518        OMX_INDEXTYPE index;
519        status_t err = mNodeInstance->getExtensionIndex(
520                "OMX.google.android.index.describeColorAspects", &index);
521        if (err == OK) {
522            // V0 dataspace
523            DescribeColorAspectsParams params;
524            InitOMXParams(&params);
525            params.nPortIndex = kPortIndexInput;
526            params.nDataSpace = mLastDataSpace;
527            params.nPixelFormat = pixelFormat;
528            params.bDataSpaceChanged = OMX_TRUE;
529            params.sAspects = mColorAspects;
530
531            err = mNodeInstance->getConfig(index, &params, sizeof(params));
532            if (err == OK) {
533                aspects = params.sAspects;
534                ALOGD("Codec resolved it to (R:%d(%s), P:%d(%s), M:%d(%s), T:%d(%s)) err=%d(%s)",
535                        params.sAspects.mRange, asString(params.sAspects.mRange),
536                        params.sAspects.mPrimaries, asString(params.sAspects.mPrimaries),
537                        params.sAspects.mMatrixCoeffs, asString(params.sAspects.mMatrixCoeffs),
538                        params.sAspects.mTransfer, asString(params.sAspects.mTransfer),
539                        err, asString(err));
540            } else {
541                params.sAspects = aspects;
542                err = OK;
543            }
544            params.bDataSpaceChanged = OMX_FALSE;
545            for (int triesLeft = 2; --triesLeft >= 0; ) {
546                status_t err = mNodeInstance->setConfig(index, &params, sizeof(params));
547                if (err == OK) {
548                    err = mNodeInstance->getConfig(index, &params, sizeof(params));
549                }
550                if (err != OK || !ColorUtils::checkIfAspectsChangedAndUnspecifyThem(
551                        params.sAspects, aspects)) {
552                    // if we can't set or get color aspects, still communicate dataspace to client
553                    break;
554                }
555
556                ALOGW_IF(triesLeft == 0, "Codec repeatedly changed requested ColorAspects.");
557            }
558        }
559
560        ALOGV("Set color aspects to (R:%d(%s), P:%d(%s), M:%d(%s), T:%d(%s)) err=%d(%s)",
561                aspects.mRange, asString(aspects.mRange),
562                aspects.mPrimaries, asString(aspects.mPrimaries),
563                aspects.mMatrixCoeffs, asString(aspects.mMatrixCoeffs),
564                aspects.mTransfer, asString(aspects.mTransfer),
565                err, asString(err));
566
567        // signal client that the dataspace has changed; this will update the output format
568        // TODO: we should tie this to an output buffer somehow, and signal the change
569        // just before the output buffer is returned to the client, but there are many
570        // ways this could fail (e.g. flushing), and we are not yet supporting this scenario.
571
572        mNodeInstance->signalEvent(
573                OMX_EventDataSpaceChanged, dataSpace,
574                (aspects.mRange << 24) | (aspects.mPrimaries << 16)
575                        | (aspects.mMatrixCoeffs << 8) | aspects.mTransfer);
576    }
577}
578
579bool GraphicBufferSource::fillCodecBuffer_l() {
580    CHECK(mExecuting && mNumFramesAvailable > 0);
581
582    if (mSuspended) {
583        return false;
584    }
585
586    int cbi = findAvailableCodecBuffer_l();
587    if (cbi < 0) {
588        // No buffers available, bail.
589        ALOGV("fillCodecBuffer_l: no codec buffers, avail now %zu",
590                mNumFramesAvailable);
591        return false;
592    }
593
594    ALOGV("fillCodecBuffer_l: acquiring buffer, avail=%zu",
595            mNumFramesAvailable);
596    BufferItem item;
597    status_t err = mConsumer->acquireBuffer(&item, 0);
598    if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
599        // shouldn't happen
600        ALOGW("fillCodecBuffer_l: frame was not available");
601        return false;
602    } else if (err != OK) {
603        // now what? fake end-of-stream?
604        ALOGW("fillCodecBuffer_l: acquireBuffer returned err=%d", err);
605        return false;
606    }
607
608    mNumBufferAcquired++;
609    mNumFramesAvailable--;
610
611    // If this is the first time we're seeing this buffer, add it to our
612    // slot table.
613    if (item.mGraphicBuffer != NULL) {
614        ALOGV("fillCodecBuffer_l: setting mBufferSlot %d", item.mSlot);
615        mBufferSlot[item.mSlot] = item.mGraphicBuffer;
616    }
617
618    if (item.mDataSpace != mLastDataSpace) {
619        onDataSpaceChanged_l(
620                item.mDataSpace, (android_pixel_format)mBufferSlot[item.mSlot]->getPixelFormat());
621    }
622
623
624    err = UNKNOWN_ERROR;
625
626    // only submit sample if start time is unspecified, or sample
627    // is queued after the specified start time
628    bool dropped = false;
629    if (mSkipFramesBeforeNs < 0ll || item.mTimestamp >= mSkipFramesBeforeNs) {
630        // if start time is set, offset time stamp by start time
631        if (mSkipFramesBeforeNs > 0) {
632            item.mTimestamp -= mSkipFramesBeforeNs;
633        }
634
635        int64_t timeUs = item.mTimestamp / 1000;
636        if (mFrameDropper != NULL && mFrameDropper->shouldDrop(timeUs)) {
637            ALOGV("skipping frame (%lld) to meet max framerate", static_cast<long long>(timeUs));
638            // set err to OK so that the skipped frame can still be saved as the lastest frame
639            err = OK;
640            dropped = true;
641        } else {
642            err = submitBuffer_l(item, cbi);
643        }
644    }
645
646    if (err != OK) {
647        ALOGV("submitBuffer_l failed, releasing bq slot %d", item.mSlot);
648        releaseBuffer(item.mSlot, item.mFrameNumber, item.mGraphicBuffer, item.mFence);
649    } else {
650        ALOGV("buffer submitted (bq %d, cbi %d)", item.mSlot, cbi);
651        setLatestBuffer_l(item, dropped);
652    }
653
654    return true;
655}
656
657bool GraphicBufferSource::repeatLatestBuffer_l() {
658    CHECK(mExecuting && mNumFramesAvailable == 0);
659
660    if (mLatestBufferId < 0 || mSuspended) {
661        return false;
662    }
663    if (mBufferSlot[mLatestBufferId] == NULL) {
664        // This can happen if the remote side disconnects, causing
665        // onBuffersReleased() to NULL out our copy of the slots.  The
666        // buffer is gone, so we have nothing to show.
667        //
668        // To be on the safe side we try to release the buffer.
669        ALOGD("repeatLatestBuffer_l: slot was NULL");
670        mConsumer->releaseBuffer(
671                mLatestBufferId,
672                mLatestBufferFrameNum,
673                EGL_NO_DISPLAY,
674                EGL_NO_SYNC_KHR,
675                mLatestBufferFence);
676        mLatestBufferId = -1;
677        mLatestBufferFrameNum = 0;
678        mLatestBufferFence = Fence::NO_FENCE;
679        return false;
680    }
681
682    int cbi = findAvailableCodecBuffer_l();
683    if (cbi < 0) {
684        // No buffers available, bail.
685        ALOGV("repeatLatestBuffer_l: no codec buffers.");
686        return false;
687    }
688
689    BufferItem item;
690    item.mSlot = mLatestBufferId;
691    item.mFrameNumber = mLatestBufferFrameNum;
692    item.mTimestamp = mRepeatLastFrameTimestamp;
693    item.mFence = mLatestBufferFence;
694
695    status_t err = submitBuffer_l(item, cbi);
696
697    if (err != OK) {
698        return false;
699    }
700
701    ++mLatestBufferUseCount;
702
703    /* repeat last frame up to kRepeatLastFrameCount times.
704     * in case of static scene, a single repeat might not get rid of encoder
705     * ghosting completely, refresh a couple more times to get better quality
706     */
707    if (--mRepeatLastFrameCount > 0) {
708        mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
709
710        if (mReflector != NULL) {
711            sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector);
712            msg->setInt32("generation", ++mRepeatLastFrameGeneration);
713            msg->post(mRepeatAfterUs);
714        }
715    }
716
717    return true;
718}
719
720void GraphicBufferSource::setLatestBuffer_l(
721        const BufferItem &item, bool dropped) {
722    ALOGV("setLatestBuffer_l");
723
724    if (mLatestBufferId >= 0) {
725        if (mLatestBufferUseCount == 0) {
726            releaseBuffer(mLatestBufferId, mLatestBufferFrameNum,
727                    mBufferSlot[mLatestBufferId], mLatestBufferFence);
728            // mLatestBufferFence will be set to new fence just below
729        }
730    }
731
732    mLatestBufferId = item.mSlot;
733    mLatestBufferFrameNum = item.mFrameNumber;
734    mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
735
736    mLatestBufferUseCount = dropped ? 0 : 1;
737    mRepeatBufferDeferred = false;
738    mRepeatLastFrameCount = kRepeatLastFrameCount;
739    mLatestBufferFence = item.mFence;
740
741    if (mReflector != NULL) {
742        sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector);
743        msg->setInt32("generation", ++mRepeatLastFrameGeneration);
744        msg->post(mRepeatAfterUs);
745    }
746}
747
748status_t GraphicBufferSource::signalEndOfInputStream() {
749    Mutex::Autolock autoLock(mMutex);
750    ALOGV("signalEndOfInputStream: exec=%d avail=%zu eos=%d",
751            mExecuting, mNumFramesAvailable, mEndOfStream);
752
753    if (mEndOfStream) {
754        ALOGE("EOS was already signaled");
755        return INVALID_OPERATION;
756    }
757
758    // Set the end-of-stream flag.  If no frames are pending from the
759    // BufferQueue, and a codec buffer is available, and we're executing,
760    // we initiate the EOS from here.  Otherwise, we'll let
761    // codecBufferEmptied() (or omxExecuting) do it.
762    //
763    // Note: if there are no pending frames and all codec buffers are
764    // available, we *must* submit the EOS from here or we'll just
765    // stall since no future events are expected.
766    mEndOfStream = true;
767
768    if (mExecuting && mNumFramesAvailable == 0) {
769        submitEndOfInputStream_l();
770    }
771
772    return OK;
773}
774
775int64_t GraphicBufferSource::getTimestamp(const BufferItem &item) {
776    int64_t timeUs = item.mTimestamp / 1000;
777
778    if (mTimePerCaptureUs > 0ll) {
779        // Time lapse or slow motion mode
780        if (mPrevCaptureUs < 0ll) {
781            // first capture
782            mPrevCaptureUs = timeUs;
783            mPrevFrameUs = timeUs;
784        } else {
785            // snap to nearest capture point
786            int64_t nFrames = (timeUs + mTimePerCaptureUs / 2 - mPrevCaptureUs)
787                    / mTimePerCaptureUs;
788            if (nFrames <= 0) {
789                // skip this frame as it's too close to previous capture
790                ALOGV("skipping frame, timeUs %lld", static_cast<long long>(timeUs));
791                return -1;
792            }
793            mPrevCaptureUs = mPrevCaptureUs + nFrames * mTimePerCaptureUs;
794            mPrevFrameUs += mTimePerFrameUs * nFrames;
795        }
796
797        ALOGV("timeUs %lld, captureUs %lld, frameUs %lld",
798                static_cast<long long>(timeUs),
799                static_cast<long long>(mPrevCaptureUs),
800                static_cast<long long>(mPrevFrameUs));
801
802        return mPrevFrameUs;
803    } else if (mMaxTimestampGapUs > 0ll) {
804        /* Cap timestamp gap between adjacent frames to specified max
805         *
806         * In the scenario of cast mirroring, encoding could be suspended for
807         * prolonged periods. Limiting the pts gap to workaround the problem
808         * where encoder's rate control logic produces huge frames after a
809         * long period of suspension.
810         */
811
812        int64_t originalTimeUs = timeUs;
813        if (mPrevOriginalTimeUs >= 0ll) {
814            if (originalTimeUs < mPrevOriginalTimeUs) {
815                // Drop the frame if it's going backward in time. Bad timestamp
816                // could disrupt encoder's rate control completely.
817                ALOGW("Dropping frame that's going backward in time");
818                return -1;
819            }
820            int64_t timestampGapUs = originalTimeUs - mPrevOriginalTimeUs;
821            timeUs = (timestampGapUs < mMaxTimestampGapUs ?
822                    timestampGapUs : mMaxTimestampGapUs) + mPrevModifiedTimeUs;
823        }
824        mPrevOriginalTimeUs = originalTimeUs;
825        mPrevModifiedTimeUs = timeUs;
826        mOriginalTimeUs.add(timeUs, originalTimeUs);
827        ALOGV("IN  timestamp: %lld -> %lld",
828            static_cast<long long>(originalTimeUs),
829            static_cast<long long>(timeUs));
830    }
831
832    return timeUs;
833}
834
835status_t GraphicBufferSource::submitBuffer_l(const BufferItem &item, int cbi) {
836    ALOGV("submitBuffer_l cbi=%d", cbi);
837
838    int64_t timeUs = getTimestamp(item);
839    if (timeUs < 0ll) {
840        return UNKNOWN_ERROR;
841    }
842
843    CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
844    codecBuffer.mGraphicBuffer = mBufferSlot[item.mSlot];
845    codecBuffer.mSlot = item.mSlot;
846    codecBuffer.mFrameNumber = item.mFrameNumber;
847
848    OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader;
849    sp<GraphicBuffer> buffer = codecBuffer.mGraphicBuffer;
850    status_t err = mNodeInstance->emptyGraphicBuffer(
851            header, buffer, OMX_BUFFERFLAG_ENDOFFRAME, timeUs,
852            item.mFence->isValid() ? item.mFence->dup() : -1);
853    if (err != OK) {
854        ALOGW("WARNING: emptyNativeWindowBuffer failed: 0x%x", err);
855        codecBuffer.mGraphicBuffer = NULL;
856        return err;
857    }
858
859    ALOGV("emptyNativeWindowBuffer succeeded, h=%p p=%p buf=%p bufhandle=%p",
860            header, header->pBuffer, buffer->getNativeBuffer(), buffer->handle);
861    return OK;
862}
863
864void GraphicBufferSource::submitEndOfInputStream_l() {
865    CHECK(mEndOfStream);
866    if (mEndOfStreamSent) {
867        ALOGV("EOS already sent");
868        return;
869    }
870
871    int cbi = findAvailableCodecBuffer_l();
872    if (cbi < 0) {
873        ALOGV("submitEndOfInputStream_l: no codec buffers available");
874        return;
875    }
876
877    // We reject any additional incoming graphic buffers, so there's no need
878    // to stick a placeholder into codecBuffer.mGraphicBuffer to mark it as
879    // in-use.
880    CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
881
882    OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader;
883    status_t err = mNodeInstance->emptyGraphicBuffer(
884            header, NULL /* buffer */, OMX_BUFFERFLAG_ENDOFFRAME | OMX_BUFFERFLAG_EOS,
885            0 /* timestamp */, -1 /* fenceFd */);
886    if (err != OK) {
887        ALOGW("emptyDirectBuffer EOS failed: 0x%x", err);
888    } else {
889        ALOGV("submitEndOfInputStream_l: buffer submitted, header=%p cbi=%d",
890                header, cbi);
891        mEndOfStreamSent = true;
892    }
893}
894
895int GraphicBufferSource::findAvailableCodecBuffer_l() {
896    CHECK(mCodecBuffers.size() > 0);
897
898    for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) {
899        if (mCodecBuffers[i].mGraphicBuffer == NULL) {
900            return i;
901        }
902    }
903    return -1;
904}
905
906int GraphicBufferSource::findMatchingCodecBuffer_l(
907        const OMX_BUFFERHEADERTYPE* header) {
908    for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) {
909        if (mCodecBuffers[i].mHeader == header) {
910            return i;
911        }
912    }
913    return -1;
914}
915
916/*
917 * Releases an acquired buffer back to the consumer for either persistent
918 * or non-persistent surfaces.
919 *
920 * id: buffer slot to release (in persistent case the id might be changed)
921 * frameNum: frame number of the frame being released
922 * buffer: GraphicBuffer pointer to release (note this must not be & as we
923 *         will clear the original mBufferSlot in persistent case)
924 * fence: fence of the frame being released
925 */
926void GraphicBufferSource::releaseBuffer(
927        int &id, uint64_t frameNum,
928        const sp<GraphicBuffer> buffer, const sp<Fence> &fence) {
929    if (mIsPersistent) {
930        mConsumer->detachBuffer(id);
931        mBufferSlot[id] = NULL;
932
933        if (mConsumer->attachBuffer(&id, buffer) == OK) {
934            mConsumer->releaseBuffer(
935                    id, 0, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
936        }
937    } else {
938        mConsumer->releaseBuffer(
939                id, frameNum, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
940    }
941    id = -1; // invalidate id
942    mNumBufferAcquired--;
943}
944
945// BufferQueue::ConsumerListener callback
946void GraphicBufferSource::onFrameAvailable(const BufferItem& /*item*/) {
947    Mutex::Autolock autoLock(mMutex);
948
949    ALOGV("onFrameAvailable exec=%d avail=%zu",
950            mExecuting, mNumFramesAvailable);
951
952    if (mEndOfStream || mSuspended) {
953        if (mEndOfStream) {
954            // This should only be possible if a new buffer was queued after
955            // EOS was signaled, i.e. the app is misbehaving.
956
957            ALOGW("onFrameAvailable: EOS is set, ignoring frame");
958        } else {
959            ALOGV("onFrameAvailable: suspended, ignoring frame");
960        }
961
962        BufferItem item;
963        status_t err = mConsumer->acquireBuffer(&item, 0);
964        if (err == OK) {
965            mNumBufferAcquired++;
966
967            // If this is the first time we're seeing this buffer, add it to our
968            // slot table.
969            if (item.mGraphicBuffer != NULL) {
970                ALOGV("onFrameAvailable: setting mBufferSlot %d", item.mSlot);
971                mBufferSlot[item.mSlot] = item.mGraphicBuffer;
972            }
973
974            releaseBuffer(item.mSlot, item.mFrameNumber,
975                    item.mGraphicBuffer, item.mFence);
976        }
977        return;
978    }
979
980    mNumFramesAvailable++;
981
982    mRepeatBufferDeferred = false;
983    ++mRepeatLastFrameGeneration;
984
985    if (mExecuting) {
986        fillCodecBuffer_l();
987    }
988}
989
990// BufferQueue::ConsumerListener callback
991void GraphicBufferSource::onBuffersReleased() {
992    Mutex::Autolock lock(mMutex);
993
994    uint64_t slotMask;
995    if (mConsumer->getReleasedBuffers(&slotMask) != NO_ERROR) {
996        ALOGW("onBuffersReleased: unable to get released buffer set");
997        slotMask = 0xffffffffffffffffULL;
998    }
999
1000    ALOGV("onBuffersReleased: 0x%016" PRIx64, slotMask);
1001
1002    for (int i = 0; i < BufferQueue::NUM_BUFFER_SLOTS; i++) {
1003        if ((slotMask & 0x01) != 0) {
1004            mBufferSlot[i] = NULL;
1005        }
1006        slotMask >>= 1;
1007    }
1008}
1009
1010// BufferQueue::ConsumerListener callback
1011void GraphicBufferSource::onSidebandStreamChanged() {
1012    ALOG_ASSERT(false, "GraphicBufferSource can't consume sideband streams");
1013}
1014
1015void GraphicBufferSource::setDefaultDataSpace(android_dataspace dataSpace) {
1016    // no need for mutex as we are not yet running
1017    ALOGD("setting dataspace: %#x", dataSpace);
1018    mConsumer->setDefaultBufferDataSpace(dataSpace);
1019    mLastDataSpace = dataSpace;
1020}
1021
1022status_t GraphicBufferSource::setRepeatPreviousFrameDelayUs(
1023        int64_t repeatAfterUs) {
1024    Mutex::Autolock autoLock(mMutex);
1025
1026    if (mExecuting || repeatAfterUs <= 0ll) {
1027        return INVALID_OPERATION;
1028    }
1029
1030    mRepeatAfterUs = repeatAfterUs;
1031
1032    return OK;
1033}
1034
1035status_t GraphicBufferSource::setMaxTimestampGapUs(int64_t maxGapUs) {
1036    Mutex::Autolock autoLock(mMutex);
1037
1038    if (mExecuting || maxGapUs <= 0ll) {
1039        return INVALID_OPERATION;
1040    }
1041
1042    mMaxTimestampGapUs = maxGapUs;
1043
1044    return OK;
1045}
1046
1047status_t GraphicBufferSource::setMaxFps(float maxFps) {
1048    Mutex::Autolock autoLock(mMutex);
1049
1050    if (mExecuting) {
1051        return INVALID_OPERATION;
1052    }
1053
1054    mFrameDropper = new FrameDropper();
1055    status_t err = mFrameDropper->setMaxFrameRate(maxFps);
1056    if (err != OK) {
1057        mFrameDropper.clear();
1058        return err;
1059    }
1060
1061    return OK;
1062}
1063
1064void GraphicBufferSource::setSkipFramesBeforeUs(int64_t skipFramesBeforeUs) {
1065    Mutex::Autolock autoLock(mMutex);
1066
1067    mSkipFramesBeforeNs =
1068            (skipFramesBeforeUs > 0) ? (skipFramesBeforeUs * 1000) : -1ll;
1069}
1070
1071status_t GraphicBufferSource::setTimeLapseConfig(const TimeLapseConfig &config) {
1072    Mutex::Autolock autoLock(mMutex);
1073
1074    if (mExecuting || config.mTimePerFrameUs <= 0ll || config.mTimePerCaptureUs <= 0ll) {
1075        return INVALID_OPERATION;
1076    }
1077
1078    mTimePerFrameUs = config.mTimePerFrameUs;
1079    mTimePerCaptureUs = config.mTimePerCaptureUs;
1080
1081    return OK;
1082}
1083
1084void GraphicBufferSource::setColorAspects(const ColorAspects &aspects) {
1085    Mutex::Autolock autoLock(mMutex);
1086    mColorAspects = aspects;
1087    ALOGD("requesting color aspects (R:%d(%s), P:%d(%s), M:%d(%s), T:%d(%s))",
1088            aspects.mRange, asString(aspects.mRange),
1089            aspects.mPrimaries, asString(aspects.mPrimaries),
1090            aspects.mMatrixCoeffs, asString(aspects.mMatrixCoeffs),
1091            aspects.mTransfer, asString(aspects.mTransfer));
1092}
1093
1094void GraphicBufferSource::onMessageReceived(const sp<AMessage> &msg) {
1095    switch (msg->what()) {
1096        case kWhatRepeatLastFrame:
1097        {
1098            Mutex::Autolock autoLock(mMutex);
1099
1100            int32_t generation;
1101            CHECK(msg->findInt32("generation", &generation));
1102
1103            if (generation != mRepeatLastFrameGeneration) {
1104                // stale
1105                break;
1106            }
1107
1108            if (!mExecuting || mNumFramesAvailable > 0) {
1109                break;
1110            }
1111
1112            bool success = repeatLatestBuffer_l();
1113
1114            if (success) {
1115                ALOGV("repeatLatestBuffer_l SUCCESS");
1116            } else {
1117                ALOGV("repeatLatestBuffer_l FAILURE");
1118                mRepeatBufferDeferred = true;
1119            }
1120            break;
1121        }
1122
1123        default:
1124            TRESPASS();
1125    }
1126}
1127
1128}  // namespace android
1129