diff --git a/include/libcamera/camera.h b/include/libcamera/camera.h index b24a29740..93a484e44 100644 --- a/include/libcamera/camera.h +++ b/include/libcamera/camera.h @@ -147,6 +147,7 @@ class Camera final : public Object, public std::enable_shared_from_this, std::unique_ptr createRequest(uint64_t cookie = 0); int queueRequest(Request *request); + int queueControls(ControlList &&controls); int start(const ControlList *controls = nullptr); int stop(); diff --git a/include/libcamera/internal/pipeline_handler.h b/include/libcamera/internal/pipeline_handler.h index b4f97477a..c25213de2 100644 --- a/include/libcamera/internal/pipeline_handler.h +++ b/include/libcamera/internal/pipeline_handler.h @@ -57,6 +57,7 @@ class PipelineHandler : public std::enable_shared_from_this, void registerRequest(Request *request); void queueRequest(Request *request); + int queueControls(Camera *camera, ControlList controls); bool completeBuffer(Request *request, FrameBuffer *buffer); void completeRequest(Request *request); @@ -76,6 +77,12 @@ class PipelineHandler : public std::enable_shared_from_this, unsigned int useCount() const { return useCount_; } virtual int queueRequestDevice(Camera *camera, Request *request) = 0; + + virtual int queueControlsDevice([[maybe_unused]] Camera *camera, [[maybe_unused]] const ControlList &controls) + { + return -EOPNOTSUPP; + } + virtual void stopDevice(Camera *camera) = 0; virtual bool acquireDevice(Camera *camera); diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp index c6481021e..7bf3d9603 100644 --- a/src/ipa/rpi/common/ipa_base.cpp +++ b/src/ipa/rpi/common/ipa_base.cpp @@ -440,6 +440,9 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms) fillDeviceStatus(params.sensorControls, ipaContext); fillSyncParams(params, ipaContext); + if (!params.requestControls.empty()) + rpiMetadata.set("ipa.request_controls", true); + if (params.buffers.embedded) { /* * Pipeline handler has supplied us with an embedded data buffer, @@ -460,7 +463,7 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms) */ AgcStatus agcStatus; bool hdrChange = false; - RPiController::Metadata &delayedMetadata = rpiMetadata_[params.delayContext]; + RPiController::Metadata &delayedMetadata = rpiMetadata_[params.delayContext % rpiMetadata_.size()]; if (!delayedMetadata.get("agc.status", agcStatus)) { rpiMetadata.set("agc.delayed_status", agcStatus); hdrChange = agcStatus.hdr.mode != hdrStatus_.mode; @@ -473,9 +476,13 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms) */ helper_->prepare(embeddedBuffer, rpiMetadata); + bool delayedRequestControls = false; + delayedMetadata.get("ipa.request_controls", delayedRequestControls); + /* Allow a 10% margin on the comparison below. */ Duration delta = (frameTimestamp - lastRunTimestamp_) * 1.0ns; - if (lastRunTimestamp_ && frameCount_ > invalidCount_ && + if (!delayedRequestControls && params.requestControls.empty() && + lastRunTimestamp_ && frameCount_ > invalidCount_ && delta < controllerMinFrameDuration * 0.9 && !hdrChange) { /* * Ensure we merge the previous frame's metadata with the current @@ -558,7 +565,7 @@ void IpaBase::processStats(const ProcessParams ¶ms) ControlList ctrls(sensorCtrls_); applyAGC(&agcStatus, ctrls, offset); rpiMetadata.set("agc.status", agcStatus); - setDelayedControls.emit(ctrls, ipaContext); + setDelayedControls.emit(ctrls, params.ipaContext); setCameraTimeoutValue(); } @@ -975,8 +982,6 @@ void IpaBase::applyControls(const ControlList &controls) /* The control provides units of microseconds. */ agc->setFixedExposureTime(0, ctrl.second.get() * 1.0us); - - libcameraMetadata_.set(controls::ExposureTime, ctrl.second.get()); break; } @@ -1000,9 +1005,6 @@ void IpaBase::applyControls(const ControlList &controls) break; agc->setFixedGain(0, ctrl.second.get()); - - libcameraMetadata_.set(controls::AnalogueGain, - ctrl.second.get()); break; } diff --git a/src/libcamera/camera.cpp b/src/libcamera/camera.cpp index 7c0e93ff4..acd519051 100644 --- a/src/libcamera/camera.cpp +++ b/src/libcamera/camera.cpp @@ -1380,6 +1380,57 @@ int Camera::queueRequest(Request *request) return 0; } +/** + * \brief Queue controls to be applied as soon as possible + * \param[in] controls The list of controls to queue + * + * This function tries to ensure that the controls in \a controls are applied + * to the camera as soon as possible. If there are still pending controls waiting + * to be applied (because of previous calls to Camera::queueControls), then + * these controls will be applied as soon as possible on a frame after those. + * + * The exact guarantees are camera dependent, but it is guaranteed that the + * controls will be applied no later than with the next \ref Request"request" + * that the application \ref Camera::queueRequest() "queues" (after any requests + * have been *used up" for sending previously queued controls). + * + * \context This function is \threadsafe. It may only be called when the camera + * is in the Running state as defined in \ref camera_operation. + * + * \return 0 on success or a negative error code otherwise + * \retval -ENODEV The camera has been disconnected from the system + * \retval -EACCES The camera is not running + */ +int Camera::queueControls(ControlList &&controls) +{ + Private *const d = _d(); + + /* + * Like requests, controls can't be queued if the camera is not running. + * Controls can be applied immediately when the camera starts using the + * Camera::Start method. + */ + + int ret = d->isAccessAllowed(Private::CameraRunning); + if (ret < 0) + return ret; + + /* + * We want to be able to queue empty control lists, as this gives a way of + * forcing another frame with the same controls as last time, before queueing + * another control list that might change them again. + */ + + patchControlList(controls); + + /* + * \todo Or `ConnectionTypeBlocking` to get the return value? + */ + d->pipe_->invokeMethod(&PipelineHandler::queueControls, ConnectionTypeQueued, this, std::move(controls)); + + return 0; +} + /** * \brief Start capture from camera * \param[in] controls Controls to be applied before starting the Camera diff --git a/src/libcamera/control_ids_rpi.yaml b/src/libcamera/control_ids_rpi.yaml index 76f4b5a1b..9bb39acfb 100644 --- a/src/libcamera/control_ids_rpi.yaml +++ b/src/libcamera/control_ids_rpi.yaml @@ -299,4 +299,21 @@ controls: This control returns performance metrics for the CNN processing stage. Two values are returned in this span, the runtime of the CNN/DNN stage and the DSP stage in milliseconds. + + - ControlListSequence: + type: int64_t + direction: out + description: | + This is the control list id number that is synchonised with this + request, meaning that this request is the first one where the images + have had the identified list of controls applied. + + Every time a control list is queued using Camera::queueControls, it + is assigned a new sequence number, starting at 1 on the first such + after the camera is started, and increasing by one on every subsequent + call. + + This is the number that is then reported back as the + ControlListSequence. When no controls have been sent with + Camera::queueControls, the ControlListSequence reports the value zero. ... diff --git a/src/libcamera/pipeline/rpi/common/delayed_controls.cpp b/src/libcamera/pipeline/rpi/common/delayed_controls.cpp index ad50a7c8b..ddd952807 100644 --- a/src/libcamera/pipeline/rpi/common/delayed_controls.cpp +++ b/src/libcamera/pipeline/rpi/common/delayed_controls.cpp @@ -119,7 +119,6 @@ DelayedControls::DelayedControls(V4L2Device *device, */ void DelayedControls::reset(unsigned int cookie) { - queueCount_ = 1; writeCount_ = 0; cookies_[0] = cookie; @@ -154,10 +153,16 @@ void DelayedControls::reset(unsigned int cookie) bool DelayedControls::push(const ControlList &controls, const unsigned int cookie) { /* Copy state from previous frame. */ - for (auto &ctrl : values_) { - Info &info = ctrl.second[queueCount_]; - info = values_[ctrl.first][queueCount_ - 1]; - info.updated = false; + if (writeCount_ > 0) { + for (auto &ctrl : values_) { + Info &info = ctrl.second[writeCount_]; + info = values_[ctrl.first][writeCount_ - 1]; + info.updated = false; + } + } else { + /* Although it works, we don't expect this. */ + LOG(RPiDelayedControls, Warning) + << "push called before applyControls"; } /* Update with new controls. */ @@ -175,18 +180,17 @@ bool DelayedControls::push(const ControlList &controls, const unsigned int cooki if (controlParams_.find(id) == controlParams_.end()) return false; - Info &info = values_[id][queueCount_]; + Info &info = values_[id][writeCount_]; info = Info(control.second); LOG(RPiDelayedControls, Debug) << "Queuing " << id->name() << " to " << info.toString() - << " at index " << queueCount_; + << " at index " << writeCount_; } - cookies_[queueCount_] = cookie; - queueCount_++; + cookies_[writeCount_] = cookie; return true; } @@ -277,12 +281,12 @@ void DelayedControls::applyControls(uint32_t sequence) } } - writeCount_ = sequence + 1; - - while (writeCount_ > queueCount_) { + while (writeCount_ < sequence + 1) { + writeCount_++; LOG(RPiDelayedControls, Debug) - << "Queue is empty, auto queue no-op."; - push({}, cookies_[queueCount_ - 1]); + << "Pushing noop with for index " << writeCount_ + << " with cookie " << cookies_[writeCount_ - 1]; + push({}, cookies_[writeCount_ - 1]); } device_->setControls(&out); diff --git a/src/libcamera/pipeline/rpi/common/delayed_controls.h b/src/libcamera/pipeline/rpi/common/delayed_controls.h index 487b0057b..48ad5a0f4 100644 --- a/src/libcamera/pipeline/rpi/common/delayed_controls.h +++ b/src/libcamera/pipeline/rpi/common/delayed_controls.h @@ -76,7 +76,6 @@ class DelayedControls std::unordered_map controlParams_; unsigned int maxDelay_; - uint32_t queueCount_; uint32_t writeCount_; std::unordered_map> values_; RingBuffer cookies_; diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp index fb8e466f6..de4539ff7 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp @@ -787,6 +787,20 @@ int PipelineHandlerBase::queueRequestDevice(Camera *camera, Request *request) return 0; } +int PipelineHandlerBase::queueControlsDevice(Camera *camera, const ControlList &controls) +{ + CameraData *data = cameraData(camera); + + /* + * For now, just hold these controls in a queue. The front item will get popped + * off and merged into the next request that we pull off our request queue and + * start to process. + */ + data->controlsQueue_.push(controls); + + return 0; +} + int PipelineHandlerBase::registerCamera(std::unique_ptr &cameraData, std::shared_ptr frontend, const std::string &frontendName, @@ -1517,4 +1531,88 @@ void CameraData::fillRequestMetadata(const ControlList &bufferControls, Request } } +static bool isControlDelayed(unsigned int id) +{ + return id == controls::ExposureTime || + id == controls::AnalogueGain || + id == controls::FrameDurationLimits || + id == controls::AeEnable || + id == controls::ExposureTimeMode || + id == controls::AnalogueGainMode; +} + +void CameraData::handleControlLists(uint32_t delayContext) +{ + /* + * If any control lists have been queued for us, merge them into the + * next request that we're going to process. This route saves controls + * from being queued always with the last request, meaning they get + * picked up here more quickly. + */ + Request *request = requestQueue_.front(); + if (!controlsQueue_.empty()) { + request->controls().merge(std::move(controlsQueue_.front()), + ControlList::MergePolicy::OverwriteExisting); + controlsQueue_.pop(); + controlListId_++; + LOG(RPI, Debug) << "Popped control list " << controlListId_ << " from queue"; + } + + /* + * Record which control list corresponds to this ipaCookie. Because setDelayedControls + * now gets called by the IPA from the start of the following frame, we must record + * the previous control list id. + */ + syncTable_.emplace(SyncTableEntry{ request->sequence(), controlListId_ }); + LOG(RPI, Debug) << "Add sync table entry: sequence " << request->sequence() + << " control list id " << controlListId_; + + /* + * We know we that we added an entry for every delayContext, so we can + * find the one for this Bayer frame, and this links us to the correct + * control list. Anything ahead of "our" entry in the queue is old, so + * can be dropped. + */ + while (!syncTable_.empty() && + syncTable_.front().ipaCookie != delayContext) { + LOG(RPI, Debug) << "Pop sync entry: ipa cookie " + << syncTable_.front().ipaCookie << " control id " + << syncTable_.front().controlListId << " for job " + << delayContext; + syncTable_.pop(); + } + + if (syncTable_.empty()) + LOG(RPI, Warning) << "Unable to find ipa cookie for PFC"; + else { + LOG(RPI, Debug) << "Using sync control id " << syncTable_.front().controlListId; + requestControlId_ = syncTable_.front().controlListId; + } + + /* The control list id is reported as the "ControlListSequence" in the metadata. */ + request->_d()->metadata().set(controls::rpi::ControlListSequence, requestControlId_); + + /* + * Controls that take effect immediately (typically ISP controls) have to be + * delayed so as to synchronise with those controls that do get delayed. So we + * must remove them from the current request, and push them onto a queue so + * that they can be used later. + */ + ControlList controls = std::move(request->controls()); + immediateControls_.push({ controlListId_, request->controls() }); + for (const auto &ctrl : controls) { + if (isControlDelayed(ctrl.first)) + request->controls().set(ctrl.first, ctrl.second); + else + immediateControls_.back().controls.set(ctrl.first, ctrl.second); + } + /* "Immediate" controls that have become due are now merged back into this request. */ + while (!immediateControls_.empty() && + immediateControls_.front().controlListId <= requestControlId_) { + request->controls().merge(immediateControls_.front().controls, + ControlList::MergePolicy::OverwriteExisting); + immediateControls_.pop(); + } +} + } /* namespace libcamera */ diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.h b/src/libcamera/pipeline/rpi/common/pipeline_base.h index 6257a9346..e6d2145d0 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.h +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.h @@ -48,7 +48,7 @@ class CameraData : public Camera::Private { public: CameraData(PipelineHandler *pipe) - : Camera::Private(pipe), state_(State::Stopped), + : Camera::Private(pipe), state_(State::Stopped), controlListId_(0), startupFrameCount_(0), invalidFrameCount_(0), buffersAllocated_(false) { } @@ -130,6 +130,9 @@ class CameraData : public Camera::Private } std::queue requestQueue_; + std::queue controlsQueue_; + uint64_t controlListId_; + uint64_t requestControlId_; /* For handling digital zoom. */ IPACameraSensorInfo sensorInfo_; @@ -175,10 +178,24 @@ class CameraData : public Camera::Private ClockRecovery wallClockRecovery_; + struct SyncTableEntry { + uint32_t ipaCookie; + uint64_t controlListId; + }; + std::queue syncTable_; + + struct ImmediateControlsEntry { + uint64_t controlListId; + ControlList controls; + }; + std::queue immediateControls_; + protected: void fillRequestMetadata(const ControlList &bufferControls, Request *request); + void handleControlLists(uint32_t delayContext); + virtual void tryRunPipeline() = 0; private: @@ -221,6 +238,7 @@ class PipelineHandlerBase : public PipelineHandler void releaseDevice(Camera *camera) override; int queueRequestDevice(Camera *camera, Request *request) override; + int queueControlsDevice(Camera *camera, const ControlList &controls) override; protected: int registerCamera(std::unique_ptr &cameraData, diff --git a/src/libcamera/pipeline/rpi/pisp/pisp.cpp b/src/libcamera/pipeline/rpi/pisp/pisp.cpp index 03e32bbaf..b881dab8f 100644 --- a/src/libcamera/pipeline/rpi/pisp/pisp.cpp +++ b/src/libcamera/pipeline/rpi/pisp/pisp.cpp @@ -2311,6 +2311,9 @@ void PiSPCameraData::tryRunPipeline() fillRequestMetadata(job.sensorControls, request); + /* This sorts out synchronisation with the ControlList queue. */ + handleControlLists(job.delayContext); + /* Set our state to say the pipeline is active. */ state_ = State::Busy; @@ -2327,7 +2330,7 @@ void PiSPCameraData::tryRunPipeline() params.buffers.bayer = RPi::MaskBayerData | bayerId; params.buffers.stats = RPi::MaskStats | statsId; params.buffers.embedded = 0; - params.ipaContext = requestQueue_.front()->sequence(); + params.ipaContext = request->sequence(); params.delayContext = job.delayContext; params.sensorControls = std::move(job.sensorControls); params.requestControls = request->controls(); diff --git a/src/libcamera/pipeline/rpi/vc4/vc4.cpp b/src/libcamera/pipeline/rpi/vc4/vc4.cpp index 14dc94361..f38122349 100644 --- a/src/libcamera/pipeline/rpi/vc4/vc4.cpp +++ b/src/libcamera/pipeline/rpi/vc4/vc4.cpp @@ -984,6 +984,9 @@ void Vc4CameraData::tryRunPipeline() fillRequestMetadata(bayerFrame.controls, request); + /* This sorts out synchronisation with the ControlList queue. */ + handleControlLists(bayerFrame.delayContext); + /* Set our state to say the pipeline is active. */ state_ = State::Busy; diff --git a/src/libcamera/pipeline_handler.cpp b/src/libcamera/pipeline_handler.cpp index 5c469e5ba..a6451db35 100644 --- a/src/libcamera/pipeline_handler.cpp +++ b/src/libcamera/pipeline_handler.cpp @@ -477,6 +477,26 @@ void PipelineHandler::queueRequest(Request *request) request->_d()->prepare(300ms); } +/** + * \brief Queue controls to apply as soon as possible + * \param[in] camera The camera + * \param[in] controls The controls to apply + * + * This function tries to queue \a controls immediately to the device by + * calling queueControlsDevice(). + * + * \context This function is called from the CameraManager thread. + */ +int PipelineHandler::queueControls(Camera *camera, ControlList controls) +{ + int ret = queueControlsDevice(camera, controls); + + if (ret < 0) + LOG(Pipeline, Error) << "Failed to queue controls: " << ret; + + return ret; +} + /** * \brief Queue one requests to the device */ @@ -543,6 +563,21 @@ void PipelineHandler::doQueueRequests(Camera *camera) * \return 0 on success or a negative error code otherwise */ +/** + * \fn PipelineHandler::queueControlsDevice() + * \brief Queue controls to be applied as soon as possible + * \param[in] camera The camera + * \param[in] controls The controls to apply + * + * This function queues \a controls to \a camera so that they can be + * applied as soon as possible + * + * \context This function is called from the CameraManager thread. + * + * \return 0 on success or a negative error code otherwise + * \return -EOPNOTSUPP if control queueing is not supported + */ + /** * \brief Complete a buffer for a request * \param[in] request The request the buffer belongs to diff --git a/src/py/libcamera/py_main.cpp b/src/py/libcamera/py_main.cpp index a983ea75c..1d88864b8 100644 --- a/src/py/libcamera/py_main.cpp +++ b/src/py/libcamera/py_main.cpp @@ -239,6 +239,21 @@ PYBIND11_MODULE(_libcamera, m) } }) + .def("queue_controls", [](Camera &self, const std::unordered_map &controls) { + + ControlList controlList(self.controls()); + + for (const auto &[id, obj] : controls) { + auto val = pyToControlValue(obj, id->type()); + controlList.set(id->id(), val); + } + + int ret = self.queueControls(std::move(controlList)); + if (ret) + throw std::system_error(-ret, std::generic_category(), + "Failed to queue controls"); + }, py::arg("controls") = std::unordered_map()) + .def_property_readonly("streams", [](Camera &self) { py::set set; for (auto &s : self.streams()) {