24 #include <fvclassifiers/siftpp.h>
30 #include <utils/time/clock.h>
31 #include <utils/time/tracker.h>
34 #include <core/exception.h>
35 #include <core/exceptions/software.h>
36 #include <fvutils/color/colorspaces.h>
37 #include <fvutils/color/conversions.h>
38 #include <fvutils/readers/png.h>
45 namespace firevision {
69 SiftppClassifier::SiftppClassifier(
const char *object_file,
79 samplingStep_ = samplingStep;
84 threshold_ = 0.04f / levels_ / 2.0f;
85 edgeThreshold_ = 10.0f;
88 unnormalized_ = unnormalized;
96 ttc_objconv_ = tt_->
add_class(
"ObjectConvert");
97 ttc_objfeat_ = tt_->
add_class(
"ObjectFeatures");
98 ttc_imgconv_ = tt_->
add_class(
"ImageConvert");
99 ttc_imgfeat_ = tt_->
add_class(
"ImageFeatures");
100 ttc_matchin_ = tt_->
add_class(
"Matching");
101 ttc_roimerg_ = tt_->
add_class(
"MergeROIs");
115 VL::pixel_t *im_pt =
new VL::pixel_t[lwidth * lheight];
116 VL::pixel_t *start = im_pt;
118 for (
unsigned int h = 0; h < lheight; ++h) {
119 for (
unsigned int w = 0; w < lwidth; ++w) {
120 int i = (buf[h * lwidth + w]);
121 VL::pixel_t norm = VL::pixel_t(255);
122 *start++ = VL::pixel_t(i) / norm;
126 obj_img_ =
new VL::PgmBuffer();
127 obj_img_->width = lwidth;
128 obj_img_->height = lheight;
129 obj_img_->data = im_pt;
132 throw Exception(
"Could not load object file");
147 obj_features_.clear();
149 obj_num_features_ = 0;
152 sigma0_ = 1.6 * powf(2.0f, 1.0f / levels_);
154 std::cout <<
"SiftppClassifier(ctor): init scalespace" << std::endl;
156 VL::Sift sift(obj_img_->data,
167 std::cout <<
"SiftppClassifier(ctor): detect object keypoints" << std::endl;
169 sift.detectKeypoints(threshold_, edgeThreshold_);
171 obj_num_features_ = sift.keypointsEnd() - sift.keypointsBegin();
172 std::cout <<
"SiftppClassifier(ctor): computed '" << obj_num_features_ <<
"' object-keypoints"
176 sift.setNormalizeDescriptor(!unnormalized_);
177 sift.setMagnification(magnif_);
179 std::cout <<
"SiftppClassifier(ctor): run detector, compute ori and des ..." << std::endl;
181 for (VL::Sift::KeypointsConstIter iter = sift.keypointsBegin(); iter != sift.keypointsEnd();
190 VL::float_t angles[4];
193 nangles = sift.computeKeypointOrientations(angles, *iter);
196 angles[0] = VL::float_t(0);
199 feat.
descs =
new VL::float_t *[nangles];
203 for (
int a = 0; a < nangles; ++a) {
207 feat.
descs[a] =
new VL::float_t[vlen_];
208 sift.computeKeypointDescriptor(feat.
descs[a], *iter, angles[a]);
213 obj_features_.push_back(feat);
217 obj_num_features_ = obj_features_.size();
218 if (!obj_num_features_ > 0) {
219 throw Exception(
"Could not compute object features");
221 std::cout <<
"SiftppClassifier(ctor): computed '" << obj_num_features_ <<
"' features from object"
234 obj_features_.clear();
237 img_features_.clear();
248 std::list<ROI> *rv =
new std::list<ROI>();
259 std::cout <<
"SiftppClassifier(classify): copy imgdat to SIFTPP Image" << std::endl;
262 VL::pixel_t *start = im_pt;
263 for (
unsigned int h = 0; h <
_height; ++h) {
264 for (
unsigned int w = 0; w <
_width; ++w) {
266 VL::pixel_t norm = VL::pixel_t(255);
267 *start++ = VL::pixel_t(i) / norm;
271 image_ =
new VL::PgmBuffer();
274 image_->data = im_pt;
288 img_features_.clear();
289 img_num_features_ = 0;
292 std::cout <<
"SiftppClassifier(classify): init scalespace" << std::endl;
294 VL::Sift sift(image_->data,
305 std::cout <<
"SiftppClassifier(classify): detect image keypoints" << std::endl;
307 sift.detectKeypoints(threshold_, edgeThreshold_);
310 img_num_features_ = sift.keypointsEnd() - sift.keypointsBegin();
311 std::cout <<
"SiftppClassifier(classify): Extracted '" << img_num_features_ <<
"' image keypoints"
315 sift.setNormalizeDescriptor(!unnormalized_);
316 sift.setMagnification(magnif_);
318 std::cout <<
"SiftppClassifier(classify): run detector, compute ori and des ..." << std::endl;
320 for (VL::Sift::KeypointsConstIter iter = sift.keypointsBegin(); iter != sift.keypointsEnd();
329 VL::float_t angles[4];
332 nangles = sift.computeKeypointOrientations(angles, *iter);
335 angles[0] = VL::float_t(0);
338 feat.
descs =
new VL::float_t *[nangles];
342 for (
int a = 0; a < nangles; ++a) {
344 feat.
descs[a] =
new VL::float_t[vlen_];
345 sift.computeKeypointDescriptor(feat.
descs[a], *iter, angles[a]);
350 img_features_.push_back(feat);
355 img_num_features_ = img_features_.size();
361 std::cout <<
"SiftppClassifier(classify): Extracted '" << img_num_features_ <<
"' image features"
367 std::cout <<
"SiftppClassifier(classify): matching ..." << std::endl;
369 std::vector<int> matches(obj_features_.size());
371 for (
unsigned i = 0; i < obj_features_.size(); i++) {
372 int match = findMatch(obj_features_[i], img_features_);
375 std::cout <<
"SiftppClassifier(classify): Matched feature " << i
376 <<
" in object image with feature " << match <<
" in image." << std::endl;
378 ROI r((
int)(img_features_[matches[i]].key.x) - 5,
379 (
int)(img_features_[matches[i]].key.y) - 5,
393 std::cout <<
"SiftppClassifier(classify) matched '" << m <<
"' of '" << obj_features_.size()
394 <<
"' features in scene." << std::endl;
396 std::cout <<
"SiftppClassifier(classify): computing ROI" << std::endl;
401 for (
unsigned i = 0; i < matches.size(); i++) {
402 if (matches[i] != -1) {
403 if ((
int)img_features_[matches[i]].key.x < x_min)
404 x_min = (int)img_features_[matches[i]].key.x;
405 if ((
int)img_features_[matches[i]].key.y < y_min)
406 y_min = (
int)img_features_[matches[i]].key.y;
407 if ((
int)img_features_[matches[i]].key.x > x_max)
408 x_max = (
int)img_features_[matches[i]].key.x;
409 if ((
int)img_features_[matches[i]].key.y > y_max)
410 y_max = (
int)img_features_[matches[i]].key.y;
433 std::cout <<
"SiftppClassifier(classify): done ... returning '" << rv->size() <<
"' ROIs."
439 SiftppClassifier::findMatch(
const Feature &ip1,
const std::vector<Feature> &ipts)
441 double mind = 1e100, second = 1e100;
444 for (
unsigned i = 0; i < ipts.size(); i++) {
445 if (ipts[i].number_of_desc != ip1.number_of_desc)
448 for (
int j = 0; j < ip1.number_of_desc; ++j) {
449 double d = distSquare(ipts[i].descs[j], ip1.descs[j], vlen_);
455 }
else if (d < second) {
461 if (mind < 0.5 * second)
468 SiftppClassifier::distSquare(VL::float_t *v1, VL::float_t *v2,
int n)
472 dsq += (v1[n - 1] - v2[n - 1]) * (v1[n - 1] - v2[n - 1]);
Base class for exceptions in Fawkes.
void print_to_stdout()
Print results to stdout.
unsigned int add_class(std::string name)
Add a new class.
void ping_end(unsigned int cls)
End of given class task.
void ping_start(unsigned int cls)
Start of given class task.
Classifier to extract regions of interest.
unsigned int _height
Height in pixels of _src buffer.
unsigned char * _src
Source buffer, encoded as YUV422_PLANAR.
unsigned int _width
Width in pixels of _src buffer.
virtual void set_buffer(unsigned char *yuv422planar_buffer)
Set buffer that the read image should be written to.
virtual unsigned int pixel_height()
Get height of read image in pixels.
virtual colorspace_t colorspace()
Get colorspace from the just read image.
virtual void read()
Read data from file.
virtual unsigned int pixel_width()
Get width of read image in pixels.
virtual std::list< ROI > * classify()
Classify image.
virtual ~SiftppClassifier()
Destructor.
Fawkes library namespace.
VL::Sift::Keypoint key
keypoint
VL::float_t ** descs
descriptors
int number_of_desc
number of descriptors