video.hpp 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360
  1. // This file is part of OpenCV project.
  2. // It is subject to the license terms in the LICENSE file found in the top-level directory
  3. // of this distribution and at http://opencv.org/license.html.
  4. //
  5. // Copyright (C) 2020 Intel Corporation
  6. #ifndef OPENCV_GAPI_VIDEO_HPP
  7. #define OPENCV_GAPI_VIDEO_HPP
  8. #include <utility> // std::tuple
  9. #include <opencv2/gapi/gkernel.hpp>
  10. /** \defgroup gapi_video G-API Video processing functionality
  11. */
  12. namespace cv { namespace gapi {
  13. /** @brief Structure for the Kalman filter's initialization parameters.*/
  14. struct GAPI_EXPORTS KalmanParams
  15. {
  16. // initial state
  17. //! corrected state (x(k)): x(k)=x'(k)+K(k)*(z(k)-H*x'(k))
  18. Mat state;
  19. //! posteriori error estimate covariance matrix (P(k)): P(k)=(I-K(k)*H)*P'(k)
  20. Mat errorCov;
  21. // dynamic system description
  22. //! state transition matrix (A)
  23. Mat transitionMatrix;
  24. //! measurement matrix (H)
  25. Mat measurementMatrix;
  26. //! process noise covariance matrix (Q)
  27. Mat processNoiseCov;
  28. //! measurement noise covariance matrix (R)
  29. Mat measurementNoiseCov;
  30. //! control matrix (B) (Optional: not used if there's no control)
  31. Mat controlMatrix;
  32. };
  33. namespace video
  34. {
  35. using GBuildPyrOutput = std::tuple<GArray<GMat>, GScalar>;
  36. using GOptFlowLKOutput = std::tuple<cv::GArray<cv::Point2f>,
  37. cv::GArray<uchar>,
  38. cv::GArray<float>>;
  39. G_TYPED_KERNEL(GBuildOptFlowPyramid, <GBuildPyrOutput(GMat,Size,GScalar,bool,int,int,bool)>,
  40. "org.opencv.video.buildOpticalFlowPyramid")
  41. {
  42. static std::tuple<GArrayDesc,GScalarDesc>
  43. outMeta(GMatDesc,const Size&,GScalarDesc,bool,int,int,bool)
  44. {
  45. return std::make_tuple(empty_array_desc(), empty_scalar_desc());
  46. }
  47. };
  48. G_TYPED_KERNEL(GCalcOptFlowLK,
  49. <GOptFlowLKOutput(GMat,GMat,cv::GArray<cv::Point2f>,cv::GArray<cv::Point2f>,Size,
  50. GScalar,TermCriteria,int,double)>,
  51. "org.opencv.video.calcOpticalFlowPyrLK")
  52. {
  53. static std::tuple<GArrayDesc,GArrayDesc,GArrayDesc> outMeta(GMatDesc,GMatDesc,GArrayDesc,
  54. GArrayDesc,const Size&,GScalarDesc,
  55. const TermCriteria&,int,double)
  56. {
  57. return std::make_tuple(empty_array_desc(), empty_array_desc(), empty_array_desc());
  58. }
  59. };
  60. G_TYPED_KERNEL(GCalcOptFlowLKForPyr,
  61. <GOptFlowLKOutput(cv::GArray<cv::GMat>,cv::GArray<cv::GMat>,
  62. cv::GArray<cv::Point2f>,cv::GArray<cv::Point2f>,Size,GScalar,
  63. TermCriteria,int,double)>,
  64. "org.opencv.video.calcOpticalFlowPyrLKForPyr")
  65. {
  66. static std::tuple<GArrayDesc,GArrayDesc,GArrayDesc> outMeta(GArrayDesc,GArrayDesc,
  67. GArrayDesc,GArrayDesc,
  68. const Size&,GScalarDesc,
  69. const TermCriteria&,int,double)
  70. {
  71. return std::make_tuple(empty_array_desc(), empty_array_desc(), empty_array_desc());
  72. }
  73. };
  74. enum BackgroundSubtractorType
  75. {
  76. TYPE_BS_MOG2,
  77. TYPE_BS_KNN
  78. };
  79. /** @brief Structure for the Background Subtractor operation's initialization parameters.*/
  80. struct BackgroundSubtractorParams
  81. {
  82. //! Type of the Background Subtractor operation.
  83. BackgroundSubtractorType operation = TYPE_BS_MOG2;
  84. //! Length of the history.
  85. int history = 500;
  86. //! For MOG2: Threshold on the squared Mahalanobis distance between the pixel
  87. //! and the model to decide whether a pixel is well described by
  88. //! the background model.
  89. //! For KNN: Threshold on the squared distance between the pixel and the sample
  90. //! to decide whether a pixel is close to that sample.
  91. double threshold = 16;
  92. //! If true, the algorithm will detect shadows and mark them.
  93. bool detectShadows = true;
  94. //! The value between 0 and 1 that indicates how fast
  95. //! the background model is learnt.
  96. //! Negative parameter value makes the algorithm use some automatically
  97. //! chosen learning rate.
  98. double learningRate = -1;
  99. //! default constructor
  100. BackgroundSubtractorParams() {}
  101. /** Full constructor
  102. @param op MOG2/KNN Background Subtractor type.
  103. @param histLength Length of the history.
  104. @param thrshld For MOG2: Threshold on the squared Mahalanobis distance between
  105. the pixel and the model to decide whether a pixel is well described by the background model.
  106. For KNN: Threshold on the squared distance between the pixel and the sample to decide
  107. whether a pixel is close to that sample.
  108. @param detect If true, the algorithm will detect shadows and mark them. It decreases the
  109. speed a bit, so if you do not need this feature, set the parameter to false.
  110. @param lRate The value between 0 and 1 that indicates how fast the background model is learnt.
  111. Negative parameter value makes the algorithm to use some automatically chosen learning rate.
  112. */
  113. BackgroundSubtractorParams(BackgroundSubtractorType op, int histLength,
  114. double thrshld, bool detect, double lRate) : operation(op),
  115. history(histLength),
  116. threshold(thrshld),
  117. detectShadows(detect),
  118. learningRate(lRate){}
  119. };
  120. G_TYPED_KERNEL(GBackgroundSubtractor, <GMat(GMat, BackgroundSubtractorParams)>,
  121. "org.opencv.video.BackgroundSubtractor")
  122. {
  123. static GMatDesc outMeta(const GMatDesc& in, const BackgroundSubtractorParams& bsParams)
  124. {
  125. GAPI_Assert(bsParams.history >= 0);
  126. GAPI_Assert(bsParams.learningRate <= 1);
  127. return in.withType(CV_8U, 1);
  128. }
  129. };
  130. void checkParams(const cv::gapi::KalmanParams& kfParams,
  131. const cv::GMatDesc& measurement, const cv::GMatDesc& control = {});
  132. G_TYPED_KERNEL(GKalmanFilter, <GMat(GMat, GOpaque<bool>, GMat, KalmanParams)>,
  133. "org.opencv.video.KalmanFilter")
  134. {
  135. static GMatDesc outMeta(const GMatDesc& measurement, const GOpaqueDesc&,
  136. const GMatDesc& control, const KalmanParams& kfParams)
  137. {
  138. checkParams(kfParams, measurement, control);
  139. return measurement.withSize(Size(1, kfParams.transitionMatrix.rows));
  140. }
  141. };
  142. G_TYPED_KERNEL(GKalmanFilterNoControl, <GMat(GMat, GOpaque<bool>, KalmanParams)>, "org.opencv.video.KalmanFilterNoControl")
  143. {
  144. static GMatDesc outMeta(const GMatDesc& measurement, const GOpaqueDesc&, const KalmanParams& kfParams)
  145. {
  146. checkParams(kfParams, measurement);
  147. return measurement.withSize(Size(1, kfParams.transitionMatrix.rows));
  148. }
  149. };
  150. } //namespace video
  151. //! @addtogroup gapi_video
  152. //! @{
  153. /** @brief Constructs the image pyramid which can be passed to calcOpticalFlowPyrLK.
  154. @note Function textual ID is "org.opencv.video.buildOpticalFlowPyramid"
  155. @param img 8-bit input image.
  156. @param winSize window size of optical flow algorithm. Must be not less than winSize
  157. argument of calcOpticalFlowPyrLK. It is needed to calculate required
  158. padding for pyramid levels.
  159. @param maxLevel 0-based maximal pyramid level number.
  160. @param withDerivatives set to precompute gradients for the every pyramid level. If pyramid is
  161. constructed without the gradients then calcOpticalFlowPyrLK will calculate
  162. them internally.
  163. @param pyrBorder the border mode for pyramid layers.
  164. @param derivBorder the border mode for gradients.
  165. @param tryReuseInputImage put ROI of input image into the pyramid if possible. You can pass false
  166. to force data copying.
  167. @return
  168. - output pyramid.
  169. - number of levels in constructed pyramid. Can be less than maxLevel.
  170. */
  171. GAPI_EXPORTS std::tuple<GArray<GMat>, GScalar>
  172. buildOpticalFlowPyramid(const GMat &img,
  173. const Size &winSize,
  174. const GScalar &maxLevel,
  175. bool withDerivatives = true,
  176. int pyrBorder = BORDER_REFLECT_101,
  177. int derivBorder = BORDER_CONSTANT,
  178. bool tryReuseInputImage = true);
  179. /** @brief Calculates an optical flow for a sparse feature set using the iterative Lucas-Kanade
  180. method with pyramids.
  181. See @cite Bouguet00 .
  182. @note Function textual ID is "org.opencv.video.calcOpticalFlowPyrLK"
  183. @param prevImg first 8-bit input image (GMat) or pyramid (GArray<GMat>) constructed by
  184. buildOpticalFlowPyramid.
  185. @param nextImg second input image (GMat) or pyramid (GArray<GMat>) of the same size and the same
  186. type as prevImg.
  187. @param prevPts GArray of 2D points for which the flow needs to be found; point coordinates must be
  188. single-precision floating-point numbers.
  189. @param predPts GArray of 2D points initial for the flow search; make sense only when
  190. OPTFLOW_USE_INITIAL_FLOW flag is passed; in that case the vector must have the same size as in
  191. the input.
  192. @param winSize size of the search window at each pyramid level.
  193. @param maxLevel 0-based maximal pyramid level number; if set to 0, pyramids are not used (single
  194. level), if set to 1, two levels are used, and so on; if pyramids are passed to input then
  195. algorithm will use as many levels as pyramids have but no more than maxLevel.
  196. @param criteria parameter, specifying the termination criteria of the iterative search algorithm
  197. (after the specified maximum number of iterations criteria.maxCount or when the search window
  198. moves by less than criteria.epsilon).
  199. @param flags operation flags:
  200. - **OPTFLOW_USE_INITIAL_FLOW** uses initial estimations, stored in nextPts; if the flag is
  201. not set, then prevPts is copied to nextPts and is considered the initial estimate.
  202. - **OPTFLOW_LK_GET_MIN_EIGENVALS** use minimum eigen values as an error measure (see
  203. minEigThreshold description); if the flag is not set, then L1 distance between patches
  204. around the original and a moved point, divided by number of pixels in a window, is used as a
  205. error measure.
  206. @param minEigThresh the algorithm calculates the minimum eigen value of a 2x2 normal matrix of
  207. optical flow equations (this matrix is called a spatial gradient matrix in @cite Bouguet00), divided
  208. by number of pixels in a window; if this value is less than minEigThreshold, then a corresponding
  209. feature is filtered out and its flow is not processed, so it allows to remove bad points and get a
  210. performance boost.
  211. @return
  212. - GArray of 2D points (with single-precision floating-point coordinates)
  213. containing the calculated new positions of input features in the second image.
  214. - status GArray (of unsigned chars); each element of the vector is set to 1 if
  215. the flow for the corresponding features has been found, otherwise, it is set to 0.
  216. - GArray of errors (doubles); each element of the vector is set to an error for the
  217. corresponding feature, type of the error measure can be set in flags parameter; if the flow wasn't
  218. found then the error is not defined (use the status parameter to find such cases).
  219. */
  220. GAPI_EXPORTS std::tuple<GArray<Point2f>, GArray<uchar>, GArray<float>>
  221. calcOpticalFlowPyrLK(const GMat &prevImg,
  222. const GMat &nextImg,
  223. const GArray<Point2f> &prevPts,
  224. const GArray<Point2f> &predPts,
  225. const Size &winSize = Size(21, 21),
  226. const GScalar &maxLevel = 3,
  227. const TermCriteria &criteria = TermCriteria(TermCriteria::COUNT |
  228. TermCriteria::EPS,
  229. 30, 0.01),
  230. int flags = 0,
  231. double minEigThresh = 1e-4);
  232. /**
  233. @overload
  234. @note Function textual ID is "org.opencv.video.calcOpticalFlowPyrLKForPyr"
  235. */
  236. GAPI_EXPORTS std::tuple<GArray<Point2f>, GArray<uchar>, GArray<float>>
  237. calcOpticalFlowPyrLK(const GArray<GMat> &prevPyr,
  238. const GArray<GMat> &nextPyr,
  239. const GArray<Point2f> &prevPts,
  240. const GArray<Point2f> &predPts,
  241. const Size &winSize = Size(21, 21),
  242. const GScalar &maxLevel = 3,
  243. const TermCriteria &criteria = TermCriteria(TermCriteria::COUNT |
  244. TermCriteria::EPS,
  245. 30, 0.01),
  246. int flags = 0,
  247. double minEigThresh = 1e-4);
  248. /** @brief Gaussian Mixture-based or K-nearest neighbours-based Background/Foreground Segmentation Algorithm.
  249. The operation generates a foreground mask.
  250. @return Output image is foreground mask, i.e. 8-bit unsigned 1-channel (binary) matrix @ref CV_8UC1.
  251. @note Functional textual ID is "org.opencv.video.BackgroundSubtractor"
  252. @param src input image: Floating point frame is used without scaling and should be in range [0,255].
  253. @param bsParams Set of initialization parameters for Background Subtractor kernel.
  254. */
  255. GAPI_EXPORTS GMat BackgroundSubtractor(const GMat& src, const cv::gapi::video::BackgroundSubtractorParams& bsParams);
  256. /** @brief Standard Kalman filter algorithm <http://en.wikipedia.org/wiki/Kalman_filter>.
  257. @note Functional textual ID is "org.opencv.video.KalmanFilter"
  258. @param measurement input matrix: 32-bit or 64-bit float 1-channel matrix containing measurements.
  259. @param haveMeasurement dynamic input flag that indicates whether we get measurements
  260. at a particular iteration .
  261. @param control input matrix: 32-bit or 64-bit float 1-channel matrix contains control data
  262. for changing dynamic system.
  263. @param kfParams Set of initialization parameters for Kalman filter kernel.
  264. @return Output matrix is predicted or corrected state. They can be 32-bit or 64-bit float
  265. 1-channel matrix @ref CV_32FC1 or @ref CV_64FC1.
  266. @details If measurement matrix is given (haveMeasurements == true), corrected state will
  267. be returned which corresponds to the pipeline
  268. cv::KalmanFilter::predict(control) -> cv::KalmanFilter::correct(measurement).
  269. Otherwise, predicted state will be returned which corresponds to the call of
  270. cv::KalmanFilter::predict(control).
  271. @sa cv::KalmanFilter
  272. */
  273. GAPI_EXPORTS GMat KalmanFilter(const GMat& measurement, const GOpaque<bool>& haveMeasurement,
  274. const GMat& control, const cv::gapi::KalmanParams& kfParams);
  275. /** @overload
  276. The case of Standard Kalman filter algorithm when there is no control in a dynamic system.
  277. In this case the controlMatrix is empty and control vector is absent.
  278. @note Function textual ID is "org.opencv.video.KalmanFilterNoControl"
  279. @param measurement input matrix: 32-bit or 64-bit float 1-channel matrix containing measurements.
  280. @param haveMeasurement dynamic input flag that indicates whether we get measurements
  281. at a particular iteration.
  282. @param kfParams Set of initialization parameters for Kalman filter kernel.
  283. @return Output matrix is predicted or corrected state. They can be 32-bit or 64-bit float
  284. 1-channel matrix @ref CV_32FC1 or @ref CV_64FC1.
  285. @sa cv::KalmanFilter
  286. */
  287. GAPI_EXPORTS GMat KalmanFilter(const GMat& measurement, const GOpaque<bool>& haveMeasurement,
  288. const cv::gapi::KalmanParams& kfParams);
  289. //! @} gapi_video
  290. } //namespace gapi
  291. } //namespace cv
  292. namespace cv { namespace detail {
  293. template<> struct CompileArgTag<cv::gapi::video::BackgroundSubtractorParams>
  294. {
  295. static const char* tag()
  296. {
  297. return "org.opencv.video.background_substractor_params";
  298. }
  299. };
  300. } // namespace detail
  301. } // namespace cv
  302. #endif // OPENCV_GAPI_VIDEO_HPP