00001
00002
00003
00004
00005
00006
00007
00008
00027 #ifndef _ObjectDetection_h_
00028 #define _ObjectDetection_h_
00029
00030 #if defined(__MINGW32__) || defined(_MSC_VER)
00031 #define __WINDOWS__
00032 #endif
00033
00034
00035
00036
00037
00038
00039
00040
00041 #include <opencv2/core/core.hpp>
00042 #include <opencv2/imgproc/imgproc_c.h>
00043 #include <opencv2/imgproc/imgproc.hpp>
00044 #include <opencv2/highgui/highgui.hpp>
00045 #include <opencv2/flann/flann.hpp>
00046 #include <opencv2/features2d/features2d.hpp>
00047 #include <opencv2/calib3d/calib3d.hpp>
00048
00049
00050 #include <opencv2/nonfree/nonfree.hpp>
00051
00052
00053 #include "opencv2/nonfree/gpu.hpp"
00054
00055
00056 #include <opencv2/contrib/contrib.hpp>
00057
00058 #include <ctype.h>
00059 #include <stdio.h>
00060 #include <stdlib.h>
00061 #include <sys/stat.h>
00062
00063 #ifndef _MSC_VER
00064 #include <sys/time.h>
00065 #else
00066 #include <direct.h>
00067 #include <time.h>
00068 #define round(val) (floor(val + 0.5))
00069 #include <cmath>
00070 #endif // _MSC_VER
00071
00072 #ifndef __WINDOWS__
00073 #include <nanohttp/nanohttp-logging.h>
00074 #include <libcsoap/soap-server.h>
00075 #endif // __WINDOWS__
00076
00077 #include <iostream>
00078 #include <string>
00079 #include <fstream>
00080 #include <vector>
00081 #include <iterator>
00082 #include <algorithm>
00083
00084 #include "od_defs.h"
00085 #include "KeypointFilter.h"
00086 #include "SMI.h"
00087
00088 #ifndef SIZE_T_MAX
00089 #define SIZE_T_MAX (size_t)-1
00090 #endif // SIZE_T_MAX
00091
00092 using namespace std;
00093
00094 typedef enum {DET_NONE,DET_SURF,DET_GPUSURF,DET_MSER,DET_SIFT,DET_FAST,
00095 DET_STAR,DET_ORB,DET_GTFF,DET_HARRIS} detector_type;
00096 typedef enum {DES_SURF,DES_GPUSURF,DES_SIFT} desc_type;
00097 typedef enum {NO_SAVING,SEPARATE_INDICES,JOINT_INDEX,MATLAB_MODE} savetype;
00098 typedef enum {KDTREE,KMEANS} flann_index_type;
00099 typedef enum {NONE,ALL_KEYPOINTS,AFTER_HOMOGRAPHY,ANY} record_keypoints_type;
00100 typedef enum {MASK_NONE,MASK_FILE,MASK_POLYGON} mask_type;
00101
00102
00103
00104
00105
00106
00107 typedef cv::flann::GenericIndex<cvflann::L2<float> > FlannIndexType;
00108
00110 struct dbitem {
00112 size_t n;
00114 string imagename;
00116 string pathname;
00119 string objectname;
00122 string descname;
00125 string indexname;
00127 int nkps;
00130 CvSize size;
00133 double apriori;
00135 size_t nmatches;
00137 size_t nsurvivors;
00139
00140 vector<cv::KeyPoint> keypoints;
00142
00143 cv::Mat descriptors;
00145 FlannIndexType *index;
00148 vector<int> ptpairs;
00150 vector<bool> survivors;
00153 bool homography;
00155 double hmatrix[9];
00157 double ihmatrix[9];
00160 CvPoint querycorners[5];
00162 CvPoint dbitemborders[5];
00165 CvPoint dbitemcorners[5];
00167 string invcorners;
00169 CvPoint2D32f gaze;
00171 vector<pair<size_t, CvPoint2D32f> > gaze_history;
00172 };
00173
00175 struct dbitem_originalorder {
00177 bool operator() (const dbitem &a, const dbitem &b) {
00178 return a.n < b.n;
00179 }
00180 };
00181
00183 struct dbitem_matchsort {
00185 bool operator() (const dbitem &a, const dbitem &b) {
00186 return a.nmatches > b.nmatches;
00187 }
00188 };
00189
00191 struct dbitem_survivorsort {
00193 bool operator() (const dbitem &a, const dbitem &b) {
00194 return a.nsurvivors > b.nsurvivors;
00195 }
00196 };
00197
00199 struct dbitem_apriorisort {
00201 bool operator() (const dbitem &a, const dbitem &b) {
00202 return a.apriori > b.apriori;
00203 }
00204 };
00205
00207 struct jointindex {
00211
00212 cv::Mat descriptors;
00215
00216 vector<cv::KeyPoint> keypoints;
00218 FlannIndexType *index;
00220 vector<pair<size_t, size_t> > keypoint_ids;
00221 };
00222
00225 struct dbresultimage {
00227 IplImage *img;
00229 double scale;
00231 size_t width;
00233 size_t height;
00234 };
00235
00237 struct descriptor_type {
00239 desc_type name;
00241 bool root;
00243 bool norm;
00244 };
00245
00247 class ObjectDetection {
00248
00249 public:
00251 static const string& Version();
00252
00254 ObjectDetection();
00255
00258 bool Initialize(bool quiet=false);
00259
00261 bool LoadRcFile();
00262
00264 bool Interpret(const string& keystr, const string& valstr);
00265
00267 bool Interpret(const char *keqv);
00268
00270 bool SetOption(const string& keystr, const string& valstr);
00271
00273 bool SetOption(const char *keqv);
00274
00276 void SetDebug(size_t d) {
00277 debug = d;
00278 if (filter != NULL)
00279 filter->SetDebug(d);
00280 }
00281
00283 size_t GetDebug() { return debug; }
00284
00286 void SetScale(double s) { scale = s; }
00287
00289 void SetScale(const string &sstr);
00290
00292 double GetScale() {
00293 if (scale>0.0)
00294 return scale;
00295 return DEFAULT_SCALING_FACTOR;
00296 }
00297
00299 void SetNewsize(int ns) { newsize = ns; }
00300
00302 void SetBlur(bool b) { applyblur = b; }
00303
00305 void SetOutImgName(const string &o) { outimgname = o; }
00306
00308 void SetOutVidName(const string &o) { outvidname = o; }
00309
00311 void SetOutDbName(const string &o) { outdbname = o; }
00312
00314 void SetShowResults(bool s) { showresults = s; }
00315
00317 void SetShowOnlineResults(bool s) { showonlineresults = s; }
00318
00320 void SetShowOnlineDbResults(bool s) { showonlinedbresults = s; }
00321
00323 bool GetShowOnlineAnyResults() {
00324 return (showonlineresults || showonlinedbresults);
00325 }
00326
00328 void SetFlannIndexMode(flann_index_type f) { flann_index_mode = f; }
00329
00331 void SetExhaustiveSearch(bool e) { exhaustivesearch = e; }
00332
00334 void SetAnalyseOnlyBest(bool a) { analyseonlybest = a; }
00335
00337 void SetWriteMatches(bool w) { writematches = w; }
00338
00340 void SetWriteMatchesFn(const string &fn) {
00341 writematchesfn = fn;
00342 SetWriteMatches(true);
00343 }
00344
00346 void SetSecondStage(bool s) { secondstage = s; }
00347
00349 void SetUseBinaryFiles(bool u) { usebinaryfiles = u; }
00350
00352 void SetDisplayInfo(bool d) { displayinfo = d; }
00353
00355 void SetShowKeypoints(bool s) { showkeypoints = s; }
00356
00358 void SetUseMask(mask_type mt) { usemask = mt; }
00359
00361 void SetUseFileMask(const string &o) {
00362 SetUseMask(MASK_FILE);
00363 SetUseXMask(o);
00364 }
00365
00367 void SetUsePolygonMask(const string &o) {
00368 SetUseMask(MASK_POLYGON);
00369 SetUseXMask(o);
00370 }
00371
00373 void SetUseXMask(const string &o);
00374
00376 void SetUseBorders(const string &o);
00377
00380 void PossiblySetShowKeypoints(bool s) {
00381 if (showkeypoints) showkeypoints = s;
00382 }
00383
00385 void SetUsedDetector(const string &d);
00386
00388 void SetUsedDescriptor(const string &d);
00389
00391 void SetUpdateApriori(bool u) { updateapriori = u; }
00392
00394 void SetMatchingKeypointsName(const string &m) {
00395 matchingkeypointsname = m;
00396 }
00397
00399 void SetMatchingKeypointsMode(const record_keypoints_type &mkm) {
00400 matchingkeypointsmode = mkm;
00401 }
00402
00404 bool RecordMatchingKeypoints(const record_keypoints_type &mkm);
00405
00412 void SetFilter(const string &f, const string &s, size_t t);
00413
00416 bool FilterInUse() {
00417 return (filter && filter->IsInUse());
00418 }
00419
00421 bool FilterLoaded() {
00422 return (filter && filter->IsLoaded());
00423 }
00424
00427 string FilterString() {
00428 if (!filter)
00429 return "XXX";
00430 else
00431 return filter->NameString();
00432 }
00433
00435 bool DbIndicesInUse() { return dbindices_in_use; }
00436
00438 void DbIndicesInUse(bool i) { dbindices_in_use = i; }
00439
00443 bool ProcessImage(const string &fn, savetype &savemode);
00444
00446 bool ProcessImage(const string &fn, savetype &savemode, dbitem &res);
00447
00451 bool ProcessImage(IplImage *image, const string& imgfn);
00452
00454 bool ProcessImageCommonPartOne(IplImage** imgptr, IplImage** maskptr);
00455
00458 bool ProcessImageCommonPartTwo(IplImage** imgptr, IplImage** maskptr,
00459 const string& imgfn);
00460
00462 string SavemodeStr(savetype &savemode) {
00463 if (savemode == NO_SAVING) return "NO_SAVING";
00464 else if (savemode == SEPARATE_INDICES) return "SEPARATE_INDICES";
00465 else if (savemode == JOINT_INDEX) return "JOINT_INDEX";
00466 else if (savemode == MATLAB_MODE) return "MATLAB_MODE";
00467 else return "UNKNOWN_SAVEMODE";
00468 }
00469
00471 void DisplayDetExtTime();
00472
00475 void AppendDescriptors();
00476
00479 void CheckKeypointsAndDescriptors(vector<cv::KeyPoint> &keyps,
00480 cv::Mat &descs);
00481
00484 void CheckKeypointsAndDescriptors();
00485
00488 bool ShowNthResult(const IplImage *image, size_t ii, bool openwindow=true);
00489
00492 bool ShowBestResult(const IplImage *image, bool openwindow=true) {
00493 return ShowNthResult(image, 0, openwindow);
00494 }
00495
00497 bool ShowAllResults(const IplImage *image) {
00498 return ShowResultsCommon(image, true);
00499 }
00500
00502 bool ShowAllMatchingResults(const IplImage *image) {
00503 return ShowResultsCommon(image, false);
00504 }
00505
00507 bool ShowResultsCommon(const IplImage *image, bool nonmatchesalso=true);
00508
00511 bool ShowDbResults(const IplImage *image);
00512
00514 bool ShowSMIResults(IplImage *img);
00515
00519 bool InsidePolygon(CvPoint p, size_t ii);
00520
00522 bool ProcessWholeDatabase(const IplImage *image);
00523
00525 bool ProcessDatabaseUntilGoodMatch(const IplImage *image);
00526
00528 bool ProcessPrevious(const IplImage *image);
00529
00531 bool DoGazeMatch();
00532
00535 bool UpdateApriori(size_t);
00536
00540 bool ProcessCapture(int i=0, const string& fn = "");
00541
00544 bool ProcessSoap(string &port, bool dummy=false);
00545
00549 bool AnalyseImages(const string& fn);
00550
00551 #ifndef __WINDOWS__
00553 static herror_t SoapExecuteCommon(SoapCtx*, SoapCtx*, bool);
00554
00559 static herror_t SoapExecuteReal(SoapCtx *req, SoapCtx *res) {
00560 return SoapExecuteCommon(req, res, false);
00561 }
00562
00566 static herror_t SoapExecuteDummy(SoapCtx *req, SoapCtx *res) {
00567 return SoapExecuteCommon(req, res, true);
00568 }
00569
00573 static herror_t SoapStatus(SoapCtx *req, SoapCtx *res);
00574 #endif // __WINDOWS__
00575
00577 static void ProcessDummy(dbitem &dummybest);
00578
00580 static void ProcessFail(dbitem &failbest);
00581
00586 bool ProcessImageFromStatic(IplImage *image, dbitem &res);
00587
00590 void GetStatusInfoFromStatic(double&, size_t&, size_t&, time_t&);
00591
00595 bool LoadImage(IplImage **imgptr, const string& fn);
00596
00599 bool ScaleImage(IplImage** imgptr);
00600
00604 bool ScaleImage(IplImage** imgptr, double s) {
00605 bool ok = true;
00606 double tmp = scale;
00607 SetScale(s);
00608 ok = ScaleImage(imgptr);
00609 scale = tmp;
00610 return ok;
00611 }
00612
00614 bool ResizeImage(IplImage** imgptr);
00615
00617 bool CreatePolygonMask(IplImage** imgptr, IplImage** maskptr,
00618 const string& fn);
00619
00621 bool LoadPolygon(vector<CvPoint>& pts, const string& fn);
00622
00624 string DetectorString(const detector_type &dt, bool lc=false);
00625
00627 string AllDetectors();
00628
00630 string DescriptorString(const desc_type &dt, bool lc=false);
00631
00633 string AllDescriptors();
00634
00636 int UsedDescriptorSize();
00637
00640 bool Detect(const IplImage *image, IplImage **maskptr);
00641
00643 bool Extract(const IplImage *image);
00644
00650 void FlannFindPairs(const cv::Mat &dbDescriptors,
00651 vector<int>& ptpairs);
00652
00656 void FlannFindPairs(FlannIndexType *index,
00657 vector<int>& ptpairs);
00658
00661
00662
00663
00668 void FindMatches(const size_t ii, bool descmode=false);
00669
00671 void FindMatchesUsingDescriptors(const size_t ii) {
00672 return FindMatches(ii, true);
00673 }
00674
00676 void FindMatchesUsingIndex(const size_t ii) {
00677 return FindMatches(ii, false);
00678 }
00679
00682 void FindMatches();
00683
00692 bool FindHomography(const IplImage *image, const size_t ii) {
00693 return FindHomography(image, db[ii]);
00694 }
00695
00699 bool FindHomography(const IplImage *image, dbitem &dbit);
00700
00703 bool CalcHomography(const double *h, double x, double y, double &X, double &Y);
00704
00708 bool ShowKeypoints(const IplImage *image, IplImage** maskptr);
00709
00712 bool LoadData_Text(dbitem &item);
00713
00716 bool LoadData_Bin(dbitem &item);
00717
00722 FlannIndexType* LoadIndex(string &indexname, bool dbg=false);
00723
00725 bool LoadBorders(dbitem &item);
00726
00734 bool SaveData_Text(const string &fn, const vector<cv::KeyPoint> &keypoints,
00735 const cv::Mat &descriptors, bool usefilter=false,
00736 bool printsize=true);
00737
00744 bool SaveData_Bin(const string &fn, const vector<cv::KeyPoint> &keypoints,
00745 const cv::Mat &descriptors, bool usefilter=false);
00746
00757 bool CreateAndSaveIndex(vector<cv::KeyPoint> &keypoints,
00758 cv::Mat &descriptors,
00759 const string &fn, const string &descfn,
00760 bool usefilter=false);
00761
00770 bool CreateAndSaveIndex(const string &fn, const string &descfn,
00771 bool usefilter=false);
00772
00781 bool LoadDbItems(vector<vector<string> > &list, const string& fn,
00782 string& indexfn, string &dsuffix, string &path);
00783
00786 bool LoadDbAprioris(vector<double>& aprioris, string& fn);
00787
00791 void LoadDb(const string &listfn, string indexfn);
00792
00797 bool LoadDbs(const string &listfns, const string &indexfns);
00798
00802 bool LoadDbs(const vector<string> &listfns, const vector<string> &indexfns);
00803
00807 bool LoadDbsList(const string &fn);
00808
00813 void SaveDbHeader(size_t n, const string &idxname);
00814
00817 void PossiblySaveDbItem(const IplImage *image, const string &fn);
00818
00823 bool LoadImgList(vector<vector<string> > &list, const string& fn);
00824
00826 bool FileExists(const string &filename);
00827
00829 bool IsAffirmative(const char *s);
00830
00832 bool ReadSMI(const string& f) { return smi.read(f); }
00833
00834 protected:
00835
00840 static double SURF_HESSIAN_THRESHOLD;
00841
00844 static bool SURF_USE_EXTENDED;
00845
00849 static double DEFAULT_SCALING_FACTOR;
00850
00853 static size_t MATCHING_NSURVIVORS_TH;
00854
00857 static size_t MATCHING_NMATCHES_TH;
00858
00861 static size_t FLANN_NTREES;
00862
00866 static double FLANN_SIMILARITY_RATIO_JOINT;
00867
00871 static double FLANN_SIMILARITY_RATIO_SEPARATE;
00872
00875
00876 static size_t FLANN_MULTIPLE_MATCH_TH;
00877
00880 static size_t FLANN_SEARCH_CHECKS;
00881
00883 static size_t JOINT_2NDSTAGE_NIMAGES;
00884
00887 static size_t HOMOGRAPHY_NIMAGES;
00888
00891 static double HOMOGRAPHY_DETERMINANT_TH;
00892
00894 static bool HOMOGRAPHY_CHECK_ANGLES;
00895
00897 static double HOMOGRAPHY_MINLENGTH_TH;
00898
00900 static double HOMOGRAPHY_MAXLENGTH_TH;
00901
00903 static double HOMOGRAPHY_BACKPROJECTION_TH;
00904
00907 static ObjectDetection *op;
00908
00909 #ifndef __WINDOWS__
00911 static pthread_mutex_t pi_mutex;
00912 #endif // __WINDOWS__
00913
00915 static size_t n_soap_threads;
00916
00918 string rcfilename;
00919
00921 size_t debug;
00922
00924 time_t start_time;
00925
00927 size_t nqueries;
00928
00930 size_t nsoaps;
00931
00933 detector_type used_detector;
00934
00936 descriptor_type used_descriptor;
00937
00939 bool usebinaryfiles;
00940
00942 bool dbindices_in_use;
00943
00947 double scale;
00948
00951 int newsize;
00952
00955 vector<double> scales;
00956
00959 bool applyblur;
00960
00962
00963
00965
00966
00969 vector<dbitem> db;
00970
00973 vector<jointindex> dbindices;
00974
00978 jointindex dbindex;
00979
00981 CvMemStorage* storage;
00982
00984
00985 vector<cv::KeyPoint> imageKeypoints;
00986
00988
00989 cv::Mat imageDescriptors;
00990
00991
00992
00993
00994
00996 CvFont statusfont;
00997
00999 flann_index_type flann_index_mode;
01000
01003 bool exhaustivesearch;
01004
01007 bool analyseonlybest;
01008
01010 bool writematches;
01011
01013 string writematchesfn;
01014
01016 bool secondstage;
01017
01020 map<size_t, size_t> matchingkeypoints;
01021
01024 mask_type usemask;
01025
01027 pair<string,string> maskreplace;
01028
01030 bool useborders;
01031
01033 pair<string,string> borderreplace;
01034
01037 bool showresults;
01038
01042 bool showonlineresults;
01043
01047 bool showonlinedbresults;
01048
01052 dbresultimage dbresultimg;
01053
01055 bool displayinfo;
01056
01058 bool showkeypoints;
01059
01062 bool updateapriori;
01063
01065 bool preserveorder;
01066
01068 bool soap;
01069
01071 vector<dbitem> previmgs;
01072
01074 string outimgname;
01075
01077 string outvidname;
01078
01080 string outdbname;
01081
01083 record_keypoints_type matchingkeypointsmode;
01084
01086 string matchingkeypointsname;
01087
01090 KeypointFilter *filter;
01091
01093 cv::VideoWriter outputVideo;
01094
01096 size_t frame_no;
01097
01099 int ncuda;
01100
01102 pair<double,double> det_ext_time;
01103
01105 SMI smi;
01106 };
01107
01108 #endif // _ObjectDetection_h_