gcpukernel.hpp 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533
  1. // This file is part of OpenCV project.
  2. // It is subject to the license terms in the LICENSE file found in the top-level directory
  3. // of this distribution and at http://opencv.org/license.html.
  4. //
  5. // Copyright (C) 2018-2020 Intel Corporation
  6. #ifndef OPENCV_GAPI_GCPUKERNEL_HPP
  7. #define OPENCV_GAPI_GCPUKERNEL_HPP
  8. #include <functional>
  9. #include <unordered_map>
  10. #include <utility>
  11. #include <vector>
  12. #include <opencv2/core/mat.hpp>
  13. #include <opencv2/gapi/gcommon.hpp>
  14. #include <opencv2/gapi/gkernel.hpp>
  15. #include <opencv2/gapi/garg.hpp>
  16. #include <opencv2/gapi/gmetaarg.hpp>
  17. #include <opencv2/gapi/util/compiler_hints.hpp> //suppress_unused_warning
  18. #include <opencv2/gapi/util/util.hpp>
  19. // FIXME: namespace scheme for backends?
  20. namespace cv {
  21. namespace gimpl
  22. {
  23. // Forward-declare an internal class
  24. class GCPUExecutable;
  25. namespace render
  26. {
  27. namespace ocv
  28. {
  29. class GRenderExecutable;
  30. }
  31. }
  32. } // namespace gimpl
  33. namespace gapi
  34. {
  35. namespace cpu
  36. {
  37. /**
  38. * \addtogroup gapi_std_backends
  39. * @{
  40. *
  41. * @brief G-API backends available in this OpenCV version
  42. *
  43. * G-API backends play a corner stone role in G-API execution
  44. * stack. Every backend is hardware-oriented and thus can run its
  45. * kernels efficiently on the target platform.
  46. *
  47. * Backends are usually "black boxes" for G-API users -- on the API
  48. * side, all backends are represented as different objects of the
  49. * same class cv::gapi::GBackend.
  50. * User can manipulate with backends by specifying which kernels to use.
  51. *
  52. * @sa @ref gapi_hld
  53. */
  54. /**
  55. * @brief Get a reference to CPU (OpenCV) backend.
  56. *
  57. * This is the default backend in G-API at the moment, providing
  58. * broader functional coverage but losing some graph model
  59. * advantages. Provided mostly for reference and prototyping
  60. * purposes.
  61. *
  62. * @sa gapi_std_backends
  63. */
  64. GAPI_EXPORTS cv::gapi::GBackend backend();
  65. /** @} */
  66. class GOCVFunctor;
  67. //! @cond IGNORED
  68. template<typename K, typename Callable>
  69. GOCVFunctor ocv_kernel(const Callable& c);
  70. template<typename K, typename Callable>
  71. GOCVFunctor ocv_kernel(Callable& c);
  72. //! @endcond
  73. } // namespace cpu
  74. } // namespace gapi
  75. // Represents arguments which are passed to a wrapped CPU function
  76. // FIXME: put into detail?
  77. class GAPI_EXPORTS GCPUContext
  78. {
  79. public:
  80. // Generic accessor API
  81. template<typename T>
  82. const T& inArg(int input) { return m_args.at(input).get<T>(); }
  83. // Syntax sugar
  84. const cv::Mat& inMat(int input);
  85. cv::Mat& outMatR(int output); // FIXME: Avoid cv::Mat m = ctx.outMatR()
  86. const cv::Scalar& inVal(int input);
  87. cv::Scalar& outValR(int output); // FIXME: Avoid cv::Scalar s = ctx.outValR()
  88. cv::MediaFrame& outFrame(int output);
  89. template<typename T> std::vector<T>& outVecR(int output) // FIXME: the same issue
  90. {
  91. return outVecRef(output).wref<T>();
  92. }
  93. template<typename T> T& outOpaqueR(int output) // FIXME: the same issue
  94. {
  95. return outOpaqueRef(output).wref<T>();
  96. }
  97. GArg state()
  98. {
  99. return m_state;
  100. }
  101. protected:
  102. detail::VectorRef& outVecRef(int output);
  103. detail::OpaqueRef& outOpaqueRef(int output);
  104. std::vector<GArg> m_args;
  105. GArg m_state;
  106. //FIXME: avoid conversion of arguments from internal representation to OpenCV one on each call
  107. //to OCV kernel. (This can be achieved by a two single time conversions in GCPUExecutable::run,
  108. //once on enter for input and output arguments, and once before return for output arguments only
  109. std::unordered_map<std::size_t, GRunArgP> m_results;
  110. friend class gimpl::GCPUExecutable;
  111. friend class gimpl::render::ocv::GRenderExecutable;
  112. };
  113. class GAPI_EXPORTS GCPUKernel
  114. {
  115. public:
  116. // This function is a kernel's execution entry point (does the processing work)
  117. using RunF = std::function<void(GCPUContext &)>;
  118. // This function is a stateful kernel's setup routine (configures state)
  119. using SetupF = std::function<void(const GMetaArgs &, const GArgs &,
  120. GArg &, const GCompileArgs &)>;
  121. GCPUKernel();
  122. GCPUKernel(const RunF& runF, const SetupF& setupF = nullptr);
  123. RunF m_runF = nullptr;
  124. SetupF m_setupF = nullptr;
  125. bool m_isStateful = false;
  126. };
  127. // FIXME: This is an ugly ad-hoc implementation. TODO: refactor
  128. namespace detail
  129. {
  130. template<class T> struct get_in;
  131. template<> struct get_in<cv::GMat>
  132. {
  133. static cv::Mat get(GCPUContext &ctx, int idx) { return ctx.inMat(idx); }
  134. };
  135. template<> struct get_in<cv::GMatP>
  136. {
  137. static cv::Mat get(GCPUContext &ctx, int idx) { return get_in<cv::GMat>::get(ctx, idx); }
  138. };
  139. template<> struct get_in<cv::GFrame>
  140. {
  141. static cv::MediaFrame get(GCPUContext &ctx, int idx) { return ctx.inArg<cv::MediaFrame>(idx); }
  142. };
  143. template<> struct get_in<cv::GScalar>
  144. {
  145. static cv::Scalar get(GCPUContext &ctx, int idx) { return ctx.inVal(idx); }
  146. };
  147. template<typename U> struct get_in<cv::GArray<U> >
  148. {
  149. static const std::vector<U>& get(GCPUContext &ctx, int idx) { return ctx.inArg<VectorRef>(idx).rref<U>(); }
  150. };
  151. template<typename U> struct get_in<cv::GOpaque<U> >
  152. {
  153. static const U& get(GCPUContext &ctx, int idx) { return ctx.inArg<OpaqueRef>(idx).rref<U>(); }
  154. };
  155. //FIXME(dm): GArray<Mat>/GArray<GMat> conversion should be done more gracefully in the system
  156. template<> struct get_in<cv::GArray<cv::GMat> >: public get_in<cv::GArray<cv::Mat> >
  157. {
  158. };
  159. //FIXME(dm): GArray<Scalar>/GArray<GScalar> conversion should be done more gracefully in the system
  160. template<> struct get_in<cv::GArray<cv::GScalar> >: public get_in<cv::GArray<cv::Scalar> >
  161. {
  162. };
  163. //FIXME(dm): GOpaque<Mat>/GOpaque<GMat> conversion should be done more gracefully in the system
  164. template<> struct get_in<cv::GOpaque<cv::GMat> >: public get_in<cv::GOpaque<cv::Mat> >
  165. {
  166. };
  167. //FIXME(dm): GOpaque<Scalar>/GOpaque<GScalar> conversion should be done more gracefully in the system
  168. template<> struct get_in<cv::GOpaque<cv::GScalar> >: public get_in<cv::GOpaque<cv::Mat> >
  169. {
  170. };
  171. template<class T> struct get_in
  172. {
  173. static T get(GCPUContext &ctx, int idx) { return ctx.inArg<T>(idx); }
  174. };
  175. struct tracked_cv_mat{
  176. tracked_cv_mat(cv::Mat& m) : r{m}, original_data{m.data} {}
  177. cv::Mat r;
  178. uchar* original_data;
  179. operator cv::Mat& (){ return r;}
  180. void validate() const{
  181. if (r.data != original_data)
  182. {
  183. util::throw_error
  184. (std::logic_error
  185. ("OpenCV kernel output parameter was reallocated. \n"
  186. "Incorrect meta data was provided ?"));
  187. }
  188. }
  189. };
  190. template<typename... Outputs>
  191. void postprocess(Outputs&... outs)
  192. {
  193. struct
  194. {
  195. void operator()(tracked_cv_mat* bm) { bm->validate(); }
  196. void operator()(...) { }
  197. } validate;
  198. //dummy array to unfold parameter pack
  199. int dummy[] = { 0, (validate(&outs), 0)... };
  200. cv::util::suppress_unused_warning(dummy);
  201. }
  202. template<class T> struct get_out;
  203. template<> struct get_out<cv::GMat>
  204. {
  205. static tracked_cv_mat get(GCPUContext &ctx, int idx)
  206. {
  207. auto& r = ctx.outMatR(idx);
  208. return {r};
  209. }
  210. };
  211. template<> struct get_out<cv::GMatP>
  212. {
  213. static tracked_cv_mat get(GCPUContext &ctx, int idx)
  214. {
  215. return get_out<cv::GMat>::get(ctx, idx);
  216. }
  217. };
  218. template<> struct get_out<cv::GScalar>
  219. {
  220. static cv::Scalar& get(GCPUContext &ctx, int idx)
  221. {
  222. return ctx.outValR(idx);
  223. }
  224. };
  225. template<> struct get_out<cv::GFrame>
  226. {
  227. static cv::MediaFrame& get(GCPUContext &ctx, int idx)
  228. {
  229. return ctx.outFrame(idx);
  230. }
  231. };
  232. template<typename U> struct get_out<cv::GArray<U>>
  233. {
  234. static std::vector<U>& get(GCPUContext &ctx, int idx)
  235. {
  236. return ctx.outVecR<U>(idx);
  237. }
  238. };
  239. //FIXME(dm): GArray<Mat>/GArray<GMat> conversion should be done more gracefully in the system
  240. template<> struct get_out<cv::GArray<cv::GMat> >: public get_out<cv::GArray<cv::Mat> >
  241. {
  242. };
  243. // FIXME(dm): GArray<vector<U>>/GArray<GArray<U>> conversion should be done more gracefully in the system
  244. template<typename U> struct get_out<cv::GArray<cv::GArray<U>> >: public get_out<cv::GArray<std::vector<U>> >
  245. {
  246. };
  247. template<typename U> struct get_out<cv::GOpaque<U>>
  248. {
  249. static U& get(GCPUContext &ctx, int idx)
  250. {
  251. return ctx.outOpaqueR<U>(idx);
  252. }
  253. };
  254. template<typename, typename>
  255. struct OCVSetupHelper;
  256. template<typename Impl, typename... Ins>
  257. struct OCVSetupHelper<Impl, std::tuple<Ins...>>
  258. {
  259. // Using 'auto' return type and 'decltype' specifier in both 'setup_impl' versions
  260. // to check existence of required 'Impl::setup' functions.
  261. // While 'decltype' specifier accepts expression we pass expression with 'comma-operator'
  262. // where first operand of comma-operator is call attempt to desired 'Impl::setup' and
  263. // the second operand is 'void()' expression.
  264. //
  265. // SFINAE for 'Impl::setup' which accepts compile arguments.
  266. template<int... IIs>
  267. static auto setup_impl(const GMetaArgs &metaArgs, const GArgs &args,
  268. GArg &state, const GCompileArgs &compileArgs,
  269. detail::Seq<IIs...>) ->
  270. decltype(Impl::setup(detail::get_in_meta<Ins>(metaArgs, args, IIs)...,
  271. std::declval<typename std::add_lvalue_reference<
  272. std::shared_ptr<typename Impl::State>
  273. >::type
  274. >(),
  275. compileArgs)
  276. , void())
  277. {
  278. // TODO: unique_ptr <-> shared_ptr conversion ?
  279. // To check: Conversion is possible only if the state which should be passed to
  280. // 'setup' user callback isn't required to have previous value
  281. std::shared_ptr<typename Impl::State> stPtr;
  282. Impl::setup(detail::get_in_meta<Ins>(metaArgs, args, IIs)..., stPtr, compileArgs);
  283. state = GArg(stPtr);
  284. }
  285. // SFINAE for 'Impl::setup' which doesn't accept compile arguments.
  286. template<int... IIs>
  287. static auto setup_impl(const GMetaArgs &metaArgs, const GArgs &args,
  288. GArg &state, const GCompileArgs &/* compileArgs */,
  289. detail::Seq<IIs...>) ->
  290. decltype(Impl::setup(detail::get_in_meta<Ins>(metaArgs, args, IIs)...,
  291. std::declval<typename std::add_lvalue_reference<
  292. std::shared_ptr<typename Impl::State>
  293. >::type
  294. >()
  295. )
  296. , void())
  297. {
  298. // The same comment as in 'setup' above.
  299. std::shared_ptr<typename Impl::State> stPtr;
  300. Impl::setup(detail::get_in_meta<Ins>(metaArgs, args, IIs)..., stPtr);
  301. state = GArg(stPtr);
  302. }
  303. static void setup(const GMetaArgs &metaArgs, const GArgs &args,
  304. GArg& state, const GCompileArgs &compileArgs)
  305. {
  306. setup_impl(metaArgs, args, state, compileArgs,
  307. typename detail::MkSeq<sizeof...(Ins)>::type());
  308. }
  309. };
  310. // OCVCallHelper is a helper class to call stateless OCV kernels and OCV kernel functors.
  311. template<typename, typename, typename>
  312. struct OCVCallHelper;
  313. // FIXME: probably can be simplified with std::apply or analogue.
  314. template<typename Impl, typename... Ins, typename... Outs>
  315. struct OCVCallHelper<Impl, std::tuple<Ins...>, std::tuple<Outs...>>
  316. {
  317. template<typename... Inputs>
  318. struct call_and_postprocess
  319. {
  320. template<typename... Outputs>
  321. static void call(Inputs&&... ins, Outputs&&... outs)
  322. {
  323. //not using a std::forward on outs is deliberate in order to
  324. //cause compilation error, by trying to bind rvalue references to lvalue references
  325. Impl::run(std::forward<Inputs>(ins)..., outs...);
  326. postprocess(outs...);
  327. }
  328. template<typename... Outputs>
  329. static void call(Impl& impl, Inputs&&... ins, Outputs&&... outs)
  330. {
  331. impl(std::forward<Inputs>(ins)..., outs...);
  332. }
  333. };
  334. template<int... IIs, int... OIs>
  335. static void call_impl(GCPUContext &ctx, detail::Seq<IIs...>, detail::Seq<OIs...>)
  336. {
  337. //Make sure that OpenCV kernels do not reallocate memory for output parameters
  338. //by comparing it's state (data ptr) before and after the call.
  339. //This is done by converting each output Mat into tracked_cv_mat object, and binding
  340. //them to parameters of ad-hoc function
  341. call_and_postprocess<decltype(get_in<Ins>::get(ctx, IIs))...>
  342. ::call(get_in<Ins>::get(ctx, IIs)..., get_out<Outs>::get(ctx, OIs)...);
  343. }
  344. template<int... IIs, int... OIs>
  345. static void call_impl(cv::GCPUContext &ctx, Impl& impl,
  346. detail::Seq<IIs...>, detail::Seq<OIs...>)
  347. {
  348. call_and_postprocess<decltype(get_in<Ins>::get(ctx, IIs))...>
  349. ::call(impl, get_in<Ins>::get(ctx, IIs)..., get_out<Outs>::get(ctx, OIs)...);
  350. }
  351. static void call(GCPUContext &ctx)
  352. {
  353. call_impl(ctx,
  354. typename detail::MkSeq<sizeof...(Ins)>::type(),
  355. typename detail::MkSeq<sizeof...(Outs)>::type());
  356. }
  357. // NB: Same as call but calling the object
  358. // This necessary for kernel implementations that have a state
  359. // and are represented as an object
  360. static void callFunctor(cv::GCPUContext &ctx, Impl& impl)
  361. {
  362. call_impl(ctx, impl,
  363. typename detail::MkSeq<sizeof...(Ins)>::type(),
  364. typename detail::MkSeq<sizeof...(Outs)>::type());
  365. }
  366. };
  367. // OCVStCallHelper is a helper class to call stateful OCV kernels.
  368. template<typename, typename, typename>
  369. struct OCVStCallHelper;
  370. template<typename Impl, typename... Ins, typename... Outs>
  371. struct OCVStCallHelper<Impl, std::tuple<Ins...>, std::tuple<Outs...>> :
  372. OCVCallHelper<Impl, std::tuple<Ins...>, std::tuple<Outs...>>
  373. {
  374. template<typename... Inputs>
  375. struct call_and_postprocess
  376. {
  377. template<typename... Outputs>
  378. static void call(typename Impl::State& st, Inputs&&... ins, Outputs&&... outs)
  379. {
  380. Impl::run(std::forward<Inputs>(ins)..., outs..., st);
  381. postprocess(outs...);
  382. }
  383. };
  384. template<int... IIs, int... OIs>
  385. static void call_impl(GCPUContext &ctx, detail::Seq<IIs...>, detail::Seq<OIs...>)
  386. {
  387. auto& st = *ctx.state().get<std::shared_ptr<typename Impl::State>>();
  388. call_and_postprocess<decltype(get_in<Ins>::get(ctx, IIs))...>
  389. ::call(st, get_in<Ins>::get(ctx, IIs)..., get_out<Outs>::get(ctx, OIs)...);
  390. }
  391. static void call(GCPUContext &ctx)
  392. {
  393. call_impl(ctx,
  394. typename detail::MkSeq<sizeof...(Ins)>::type(),
  395. typename detail::MkSeq<sizeof...(Outs)>::type());
  396. }
  397. };
  398. } // namespace detail
  399. template<class Impl, class K>
  400. class GCPUKernelImpl: public cv::detail::KernelTag
  401. {
  402. using CallHelper = cv::detail::OCVCallHelper<Impl, typename K::InArgs, typename K::OutArgs>;
  403. public:
  404. using API = K;
  405. static cv::gapi::GBackend backend() { return cv::gapi::cpu::backend(); }
  406. static cv::GCPUKernel kernel() { return GCPUKernel(&CallHelper::call); }
  407. };
  408. template<class Impl, class K, class S>
  409. class GCPUStKernelImpl: public cv::detail::KernelTag
  410. {
  411. using StSetupHelper = detail::OCVSetupHelper<Impl, typename K::InArgs>;
  412. using StCallHelper = detail::OCVStCallHelper<Impl, typename K::InArgs, typename K::OutArgs>;
  413. public:
  414. using API = K;
  415. using State = S;
  416. static cv::gapi::GBackend backend() { return cv::gapi::cpu::backend(); }
  417. static cv::GCPUKernel kernel() { return GCPUKernel(&StCallHelper::call,
  418. &StSetupHelper::setup); }
  419. };
  420. #define GAPI_OCV_KERNEL(Name, API) struct Name: public cv::GCPUKernelImpl<Name, API>
  421. // TODO: Reuse Anatoliy's logic for support of types with commas in macro.
  422. // Retrieve the common part from Anatoliy's logic to the separate place.
  423. #define GAPI_OCV_KERNEL_ST(Name, API, State) \
  424. struct Name: public cv::GCPUStKernelImpl<Name, API, State> \
  425. class gapi::cpu::GOCVFunctor : public gapi::GFunctor
  426. {
  427. public:
  428. using Impl = std::function<void(GCPUContext &)>;
  429. using Meta = cv::GKernel::M;
  430. GOCVFunctor(const char* id, const Meta &meta, const Impl& impl)
  431. : gapi::GFunctor(id), impl_{GCPUKernel(impl), meta}
  432. {
  433. }
  434. GKernelImpl impl() const override { return impl_; }
  435. gapi::GBackend backend() const override { return gapi::cpu::backend(); }
  436. private:
  437. GKernelImpl impl_;
  438. };
  439. //! @cond IGNORED
  440. template<typename K, typename Callable>
  441. gapi::cpu::GOCVFunctor gapi::cpu::ocv_kernel(Callable& c)
  442. {
  443. using P = cv::detail::OCVCallHelper<Callable, typename K::InArgs, typename K::OutArgs>;
  444. return GOCVFunctor{ K::id()
  445. , &K::getOutMeta
  446. , std::bind(&P::callFunctor, std::placeholders::_1, std::ref(c))
  447. };
  448. }
  449. template<typename K, typename Callable>
  450. gapi::cpu::GOCVFunctor gapi::cpu::ocv_kernel(const Callable& c)
  451. {
  452. using P = cv::detail::OCVCallHelper<Callable, typename K::InArgs, typename K::OutArgs>;
  453. return GOCVFunctor{ K::id()
  454. , &K::getOutMeta
  455. , std::bind(&P::callFunctor, std::placeholders::_1, c)
  456. };
  457. }
  458. //! @endcond
  459. } // namespace cv
  460. #endif // OPENCV_GAPI_GCPUKERNEL_HPP