41 #include <boost/assert.hpp>
42 #include <util/Stride.h>
49 #undef OSVR_DEBUG_ERROR_VARIANCE_WHEN_TRACKING_LOST
50 #undef OSVR_DEBUG_ERROR_VARIANCE
52 #undef OSVR_UVBI_FRAMEDROP_HEURISTIC_WARNING
54 #define OSVR_VERBOSE_ERROR_BOUNDS
57 #undef OSVR_UVBI_DUMP_BLOB_CSV
61 enum class TargetTrackingState {
66 RANSACWhenBlobDetected,
67 RANSACKalmanWhenBlobDetected
70 enum class TargetHealthState {
72 StopTrackingErrorBoundsExceeded,
73 StopTrackingLostSight,
74 HardResetNonFiniteState
77 static const auto MAX_FRAMES_WITHOUT_BEACONS = 150;
79 inline double getLimitOnMaxPositionalErrorVariance(
double distance) {
88 static const double A = 6.4116155459;
89 static const double B = 8e-5;
90 return B * std::exp(A * distance);
93 inline double getMaxPositionalErrorVariance(BodyState
const &bodyState) {
94 return bodyState.errorCovariance().diagonal().head<3>().maxCoeff();
99 inline bool isStateSCAAT(TargetTrackingState trackingState) {
100 return !(trackingState == TargetTrackingState::RANSAC ||
101 trackingState == TargetTrackingState::RANSACKalman);
106 TargetHealthState operator()(
BodyState const &bodyState,
107 LedPtrList
const &leds,
108 TargetTrackingState trackingState) {
109 if (!bodyState.
stateVector().array().allFinite()) {
110 return TargetHealthState::HardResetNonFiniteState;
112 if (!bodyState.getQuaternion().coeffs().array().allFinite()) {
113 return TargetHealthState::HardResetNonFiniteState;
116 m_framesWithoutValidBeacons++;
118 m_framesWithoutValidBeacons = 0;
121 if (isStateSCAAT(trackingState) &&
122 m_framesWithoutValidBeacons != 0) {
123 auto maxPositionalError =
124 getMaxPositionalErrorVariance(bodyState);
125 auto distance = bodyState.position().z();
127 getLimitOnMaxPositionalErrorVariance(distance);
128 if (maxPositionalError > errorLimit) {
129 return TargetHealthState::StopTrackingErrorBoundsExceeded;
133 if (m_framesWithoutValidBeacons > MAX_FRAMES_WITHOUT_BEACONS) {
134 return TargetHealthState::StopTrackingLostSight;
136 return TargetHealthState::OK;
140 std::size_t m_framesWithoutValidBeacons = 0;
145 : bodyInterface(bodyIface), kalmanEstimator(params),
150 #ifdef OSVR_UVBI_DUMP_BLOB_CSV
152 blobFile(
"blobs.csv"), csv(blobFile)
153 #endif // OSVR_UVBI_DUMP_BLOB_CSV
158 LedPtrList usableLeds;
159 LedIdentifierPtr identifier;
166 TargetTrackingState trackingState = TargetTrackingState::RANSAC;
167 TargetTrackingState lastFrameAlgorithm = TargetTrackingState::RANSAC;
176 bool hasPrev =
false;
182 std::ostringstream outputSink;
184 #ifdef OSVR_UVBI_DUMP_BLOB_CSV
185 std::ofstream blobFile;
187 #endif // OSVR_UVBI_DUMP_BLOB_CSV
190 inline BeaconStateVec createBeaconStateVec(
ConfigParams const ¶ms,
192 Eigen::Vector3d &beaconOffset) {
196 Eigen::Vector3d beaconSum = Eigen::Vector3d::Zero();
197 auto bNum =
size_t{0};
198 for (
auto &beacon : setupData.locations) {
199 beaconSum += cvToVector(beacon).cast<
double>();
202 beaconOffset = beaconSum / bNum;
204 std::cout <<
"[Tracker Target] Computed beacon centroid: "
205 << beaconOffset.transpose() << std::endl;
213 using size_type = TargetSetupData::size_type;
214 const auto n = setupData.numBeacons();
215 BeaconStateVec beacons;
217 Eigen::Vector3d location;
218 for (size_type i = 0; i < n; ++i) {
219 location = cvToVector(setupData.locations[i]).cast<
double>() -
222 location, Eigen::Vector3d::Constant(
223 setupData.initialAutocalibrationErrors[i])
225 beacons.emplace_back(std::move(beacon));
232 Eigen::Vector3d
const &targetToBody,
235 : m_body(body), m_id(id), m_targetToBody(targetToBody),
236 m_numBeacons(setupData.numBeacons()),
237 m_beaconMeasurementVariance(setupData.baseMeasurementVariances),
238 m_beaconFixed(setupData.isFixed),
239 m_beaconEmissionDirection(setupData.emissionDirections),
240 m_impl(new
Impl(getParams(), bodyIface)) {
244 createBeaconStateVec(getParams(), setupData, m_beaconOffset);
247 for (
auto &beacon : m_beacons) {
248 m_origBeacons.emplace_back(
new BeaconState(*beacon));
252 m_beaconDebugData.resize(m_beacons.size());
254 #ifdef OSVR_UVBI_DUMP_BLOB_CSV
258 for (std::size_t i = 0; i < m_numBeacons; ++i) {
259 auto prefix = std::to_string(i + 1) +
".";
260 m_impl->csv.getColumn(prefix +
"x");
261 m_impl->csv.getColumn(prefix +
"y");
262 m_impl->csv.getColumn(prefix +
"diameter");
263 m_impl->csv.getColumn(prefix +
"area");
264 m_impl->csv.getColumn(prefix +
"bright");
266 m_impl->csv.startOutput();
268 #endif // OSVR_UVBI_DUMP_BLOB_CSV
271 std::unique_ptr<OsvrHdkLedIdentifier> identifier(
273 m_impl->identifier = std::move(identifier);
275 m_verifyInvariants();
278 TrackedBodyTarget::~TrackedBodyTarget() {}
281 return BodyTargetId(getBody().
getId(),
getId());
286 BOOST_ASSERT(!i.empty());
287 BOOST_ASSERT_MSG(i.value() < getNumBeacons(),
288 "Beacon ID must be less than number of beacons.");
289 BOOST_ASSERT_MSG(i.value() >= 0,
290 "Beacon ID must not be a sentinel value!");
291 return m_beacons.at(i.value())->stateVector() + m_beaconOffset;
296 BOOST_ASSERT(!i.empty());
297 BOOST_ASSERT_MSG(i.value() < getNumBeacons(),
298 "Beacon ID must be less than number of beacons.");
299 BOOST_ASSERT_MSG(i.value() >= 0,
300 "Beacon ID must not be a sentinel value!");
301 return m_beacons.at(i.value())->errorCovariance().diagonal();
306 for (
auto &beacon : m_origBeacons) {
312 LedMeasurementVec
const &undistortedLeds) {
315 LedMeasurementVec measurements{undistortedLeds};
316 const auto prevUsableLedCount =
usableLeds().size();
321 if (getParams().streamBeaconDebugInfo) {
324 for (
auto &data : m_beaconDebugData) {
331 auto &myLeds = m_impl->leds;
333 const auto prevLedCount = myLeds.size();
335 const auto numMeasurements = measurements.size();
338 m_numBeacons, blobMoveThreshold);
341 static const auto HEAP_PREFIX =
"[ASSIGN HEAP] ";
342 bool verbose =
false;
343 if (getParams().extraVerbose) {
345 static ::util::Stride assignStride(157);
352 std::cout << HEAP_PREFIX <<
"Heap contains " << assignment.
size()
353 <<
" elts, of possible "
358 auto ledAndMeasurement = assignment.
getMatch();
359 auto &led = ledAndMeasurement.first;
360 auto &meas = ledAndMeasurement.second;
361 led.addMeasurement(meas, blobsKeepIdentity);
362 if (handleOutOfRangeIds(led, m_numBeacons)) {
368 std::cerr <<
"ERROR: We just got a faulty one: filtering in "
371 <<
" made an LED go invalid. The measurement "
372 << (success ?
"could" :
"could NOT")
373 <<
" be resubmitted successfully\n";
377 const auto numUnclaimedLedObjects =
378 assignment.numUnclaimedLedObjects();
379 const auto numUnclaimedMeasurements =
380 assignment.numUnclaimedMeasurements();
381 const auto usedMeasurements =
382 numMeasurements - numUnclaimedMeasurements;
383 if (usedMeasurements != assignment.numCompletedMatches()) {
386 <<
"Error: numMeasurements - numUnclaimedMeasurements = "
387 << usedMeasurements <<
" but object reports "
388 << assignment.numCompletedMatches() <<
" matches!\n";
392 <<
"Matched: " << assignment.numCompletedMatches()
393 <<
"\tUnclaimed Meas: " << assignment.numUnclaimedMeasurements()
394 <<
"\tUnclaimed LED: " << assignment.numUnclaimedLedObjects()
396 <<
"\tRemaining: " << assignment.
size() <<
"\n";
406 myLeds.emplace_back(m_impl->identifier.get(), meas);
413 #ifdef OSVR_UVBI_DUMP_BLOB_CSV
415 static bool first =
true;
418 std::cout <<
"Dumping first row of blob data." << std::endl;
420 auto &row = m_impl->csv.row();
423 std::to_string(led->getOneBasedID().value()) +
".";
425 prefix, cvToVector(led->getLocationForTracking()))
427 led->getMeasurement().diameter)
428 <<
util::cell(prefix +
"area", led->getMeasurement().area)
429 <<
util::cell(prefix +
"bright", led->isBright());
432 #endif // OSVR_UVBI_DUMP_BLOB_CSV
434 #ifdef OSVR_UVBI_FRAMEDROP_HEURISTIC_WARNING
435 if (
usableLeds().empty() && prevUsableLedCount > 3 &&
436 assignment.numCompletedMatches() > prevUsableLedCount / 2) {
440 msg() <<
"WARNING: Likely dropped camera frame (reduce CPU load!): "
441 "no usable (identified) LEDs this frame out of "
443 <<
" detected likely beacons, while last frame contained "
444 << prevUsableLedCount <<
" usable LEDs." << std::endl;
448 return assignment.numCompletedMatches();
459 bool validStateAndTime) {
466 bool permitKalman = m_impl->permitKalman && validStateAndTime;
471 if (!m_hasPoseEstimate && isStateSCAAT(m_impl->trackingState)) {
477 switch (m_impl->healthEval(bodyState,
usableLeds(),
478 m_impl->trackingState)) {
479 case TargetHealthState::StopTrackingErrorBoundsExceeded: {
480 msg() <<
"In flight reset - error bounds exceeded...";
481 #ifdef OSVR_VERBOSE_ERROR_BOUNDS
482 auto maxPositionalError =
483 getMaxPositionalErrorVariance(getBody().getState());
484 auto distance = getBody().getState().position().z();
485 auto errorLimit = getLimitOnMaxPositionalErrorVariance(distance);
486 std::cout <<
" [" << maxPositionalError <<
"\t > " << errorLimit
487 <<
"\t (@ " << distance <<
"m)]";
489 std::cout << std::endl;
491 if (m_impl->softResets) {
493 enterRANSACKalmanMode();
499 case TargetHealthState::StopTrackingLostSight:
501 msg() <<
"Lost sight of beacons for too long, awaiting their "
507 case TargetHealthState::HardResetNonFiniteState:
508 msg() <<
"Hard reset - non-finite target state." << std::endl;
511 case TargetHealthState::OK:
516 switch (m_impl->trackingState) {
517 case TargetTrackingState::RANSACWhenBlobDetected: {
520 <<
"In flight reset - beacons detected, re-acquiring fix..."
527 case TargetTrackingState::RANSACKalmanWhenBlobDetected: {
530 <<
"In flight reset - beacons detected, re-acquiring fix..."
532 enterRANSACKalmanMode();
543 camParams, m_beacons, m_beaconMeasurementVariance, m_beaconFixed,
544 m_beaconEmissionDirection, startingTime, bodyState,
545 getBody().getProcessModel(), m_beaconDebugData,
547 Eigen::Vector3d::Zero()};
548 switch (m_impl->trackingState) {
549 case TargetTrackingState::RANSAC: {
550 m_hasPoseEstimate = m_impl->ransacEstimator(params,
usableLeds());
551 m_impl->lastFrameAlgorithm = TargetTrackingState::RANSAC;
555 case TargetTrackingState::RANSACKalman: {
557 m_impl->ransacKalmanEstimator(params,
usableLeds(), tv);
558 m_impl->lastFrameAlgorithm = TargetTrackingState::RANSACKalman;
562 case TargetTrackingState::RANSACWhenBlobDetected:
563 case TargetTrackingState::EnteringKalman:
564 case TargetTrackingState::Kalman: {
568 m_impl->kalmanEstimator(params,
usableLeds(), tv, videoDt);
569 m_impl->lastFrameAlgorithm = TargetTrackingState::Kalman;
576 #ifdef OSVR_DEBUG_ERROR_VARIANCE
578 static ::util::Stride varianceStride(101);
579 if (++varianceStride) {
580 msg() <<
"Max positional error variance: "
581 << getMaxPositionalErrorVariance(getBody().getState())
582 <<
" Distance: " << getBody().getState().position().z()
588 switch (m_impl->trackingState) {
589 case TargetTrackingState::RANSACKalman:
590 case TargetTrackingState::RANSAC: {
591 if (m_hasPoseEstimate && permitKalman) {
596 case TargetTrackingState::EnteringKalman:
597 m_impl->trackingState = TargetTrackingState::Kalman;
600 case TargetTrackingState::Kalman: {
601 #ifndef OSVR_RANSACKALMAN
602 auto health = m_impl->kalmanEstimator.getTrackingHealth();
604 case SCAATKalmanPoseEstimator::TrackingHealth::NeedsHardResetNow:
605 msg() <<
"In flight reset - lost fix..." << std::endl;
608 case SCAATKalmanPoseEstimator::TrackingHealth::
609 SoftResetWhenBeaconsSeen:
610 #ifdef OSVR_DEBUG_ERROR_VARIANCE_WHEN_TRACKING_LOST
611 msg() <<
"Max positional error variance: "
612 << getMaxPositionalErrorVariance(getBody().getState())
615 if (m_impl->softResets) {
616 m_impl->trackingState =
617 TargetTrackingState::RANSACKalmanWhenBlobDetected;
619 m_impl->trackingState =
620 TargetTrackingState::RANSACWhenBlobDetected;
623 case SCAATKalmanPoseEstimator::TrackingHealth::Functioning:
636 m_impl->lastEstimate = tv;
641 return m_hasPoseEstimate;
646 Eigen::Quaterniond &quat,
int skipBrightsCutoff,
647 std::size_t iterations) {
649 Eigen::Vector3d outXlate;
650 Eigen::Quaterniond outQuat;
651 auto gotPose = m_impl->ransacEstimator(
652 camParams,
usableLeds(), m_beacons, m_beaconDebugData, outXlate,
653 outQuat, skipBrightsCutoff, iterations);
656 xlate = outXlate + computeTranslationCorrectionToBody(outQuat);
667 return computeTranslationCorrection(
668 m_beaconOffset, m_impl->bodyInterface.state.getQuaternion());
670 return computeTranslationCorrectionToBody(
671 m_impl->bodyInterface.state.getQuaternion());
674 Eigen::Vector3d TrackedBodyTarget::computeTranslationCorrection(
675 Eigen::Vector3d
const &bodyFrameOffset,
676 Eigen::Quaterniond
const &orientation) {
677 Eigen::Vector3d ret = (Eigen::Isometry3d(orientation) *
678 Eigen::Translation3d(bodyFrameOffset))
683 Eigen::Vector3d TrackedBodyTarget::computeTranslationCorrectionToBody(
684 Eigen::Quaterniond
const &orientation)
const {
685 return computeTranslationCorrection(m_beaconOffset + m_targetToBody,
689 ConfigParams
const &TrackedBodyTarget::getParams()
const {
692 std::ostream &TrackedBodyTarget::msg()
const {
693 if (getParams().silent) {
694 m_impl->outputSink.str(
"");
695 return m_impl->outputSink;
697 return std::cout <<
"[Tracker Target " <<
getQualifiedId() <<
"] ";
699 void TrackedBodyTarget::enterKalmanMode() {
700 msg() <<
"Entering SCAAT Kalman mode..." << std::endl;
701 m_impl->trackingState = TargetTrackingState::EnteringKalman;
702 m_impl->kalmanEstimator.resetCounters();
705 void TrackedBodyTarget::enterRANSACMode() {
707 #ifndef OSVR_UVBI_ASSUME_SINGLE_TARGET_PER_BODY
709 "We may not be able/willing to run right over the body velocity just because this target lost its fix"
711 #ifdef OSVR_DEBUG_ERROR_VARIANCE_WHEN_TRACKING_LOST
712 msg() <<
"Max positional error variance: "
713 << getMaxPositionalErrorVariance(getBody().getState())
716 m_impl->trackingResets++;
718 switch (m_impl->trackingState) {
719 case TargetTrackingState::RANSACWhenBlobDetected:
720 case TargetTrackingState::Kalman:
721 getBody().getState().angularVelocity() = Eigen::Vector3d::Zero();
722 getBody().getState().velocity() = Eigen::Vector3d::Zero();
724 case TargetTrackingState::EnteringKalman:
730 m_impl->trackingState = TargetTrackingState::RANSAC;
733 void TrackedBodyTarget::enterRANSACKalmanMode() {
735 m_impl->trackingResets++;
738 switch (m_impl->trackingState) {
739 case TargetTrackingState::RANSACWhenBlobDetected:
740 case TargetTrackingState::Kalman:
741 getBody().getState().angularVelocity() = Eigen::Vector3d::Zero();
742 getBody().getState().velocity() = Eigen::Vector3d::Zero();
744 case TargetTrackingState::EnteringKalman:
751 msg() <<
"Soft reset as configured..." << std::endl;
752 m_impl->trackingState = TargetTrackingState::RANSACKalman;
758 return m_impl->usableLeds;
762 return m_impl->trackingResets;
765 inline std::ptrdiff_t getNumUsedLeds(LedPtrList
const &usableLeds) {
766 return std::count_if(
767 usableLeds.begin(), usableLeds.end(),
768 [](
Led const *ledPtr) {
return ledPtr->wasUsedLastFrame(); });
772 TargetStatusMeasurement measurement)
const {
773 switch (measurement) {
774 case TargetStatusMeasurement::MaxPosErrorVariance:
775 return getMaxPositionalErrorVariance(getBody().getState());
776 case TargetStatusMeasurement::PosErrorVarianceLimit: {
777 auto distance = getBody().getState().position().z();
778 return getLimitOnMaxPositionalErrorVariance(distance);
780 case TargetStatusMeasurement::NumUsableLeds:
781 return static_cast<double>(
usableLeds().size());
782 case TargetStatusMeasurement::NumUsedLeds:
783 return static_cast<double>(getNumUsedLeds(
usableLeds()));
794 void TrackedBodyTarget::updateUsableLeds() {
797 auto &
leds = m_impl->leds;
798 for (
auto &led :
leds) {
799 if (!led.identified()) {
802 usable.push_back(&led);
806 TrackedBodyTarget::getLastUpdate()
const {
807 return m_impl->lastEstimate;
810 void TrackedBodyTarget::dumpBeaconsToConsole()
const {
813 auto numBeacons = getNumBeacons();
814 std::cout <<
"BeaconsID,x,y,z" << std::endl;
815 Eigen::IOFormat ourFormat(Eigen::StreamPrecision, 0,
",");
816 for (UnderlyingBeaconIdType i = 0; i < numBeacons; ++i) {
817 auto id = ZeroBasedBeaconId(i);
818 std::cout << i + 1 <<
","
Helper class to keep track of the state of a blob over time. This is used to help determine the ident...
OSVR_EXTERN_C_END double osvrTimeValueDurationSeconds(const OSVR_TimeValue *tvA, const OSVR_TimeValue *tvB)
Compute the difference between the two time values, returning the duration as a double-precision floa...
TargetId getId() const
Get the target id within this body.
bool uncalibratedRANSACPoseEstimateFromLeds(CameraParameters const &camParams, Eigen::Vector3d &xlate, Eigen::Quaterniond &quat, int skipBrightsCutoff=-1, std::size_t iterations=5)
StateVector const & stateVector() const
xhat
A way for targets to access internals of a tracked body.
void eraseUnclaimedLedObjects(bool verbose=false)
size_type theoreticalMaxSize() const
Eigen::Vector3d getBeaconAutocalibPosition(ZeroBasedBeaconId i) const
double manualBeaconOffset[3]
std::size_t processLedMeasurements(LedMeasurementVec const &undistortedLeds)
bool permitKalman
Permit as a purely policy measure.
bool updatePoseEstimateFromLeds(CameraParameters const &camParams, osvr::util::time::TimeValue const &tv, BodyState &bodyState, osvr::util::time::TimeValue const &startingTime, bool validStateAndTime)
ConfigParams const & getParams() const
Header file for class that tracks and identifies LEDs.
detail::CellGroup< T > cellGroup(const char *groupHeader, T const &data)
Helper function to create a cell group with a group header prefix.
void populateStructures()
Must call first, and only once.
bool resumbitMeasurement(LedMeasurement const &meas)
std::size_t trackingResets
LedPtrList const & usableLeds() const
Get a list of pointers to all recognized, in-range beacons/leds.
double heapSizeFraction() const
The fraction of the theoretical max that the size is.
LedGroup const & leds() const
Get all beacons/leds, including unrecognized ones.
General configuration parameters.
double getInternalStatusMeasurement(TargetStatusMeasurement measurement) const
size_type size() const
Entries in the heap of possibilities.
void forEachUnclaimedMeasurement(F &&op)
TrackedBodyTarget(TrackedBody &body, BodyTargetInterface const &bodyIface, Eigen::Vector3d const &targetToBody, TargetSetupData const &setupData, TargetId id)
bool hasMoreMatches(bool verbose=false)
bool debug
Whether to show the debug windows and debug messages.
BodyTargetId getQualifiedId() const
Get a fully-qualified (within a tracking system) id for this target.
std::size_t numTrackingResets() const
double softResetPositionVarianceScale
LedAndMeasurement getMatch(bool verbose=false)
Requires that hasMoreMatches() has been run and returns true.
Standardized, portable parallel to struct timeval for representing both absolute times and time inter...
Eigen::Vector3d getStateCorrection() const
Get the beacon offset transformed into world space.
detail::Cell< T > cell(const char *header, T const &data)
Helper free function to make a CSV cell.
void resetBeaconAutocalib()
Reset beacon autocalibration position and variance.
double softResetOrientationVariance
Soft reset data incorporation parameter: Orientation variance.
Eigen::Vector3d getBeaconAutocalibVariance(ZeroBasedBeaconId i) const