gcomputation.hpp 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575
  1. // This file is part of OpenCV project.
  2. // It is subject to the license terms in the LICENSE file found in the top-level directory
  3. // of this distribution and at http://opencv.org/license.html.
  4. //
  5. // Copyright (C) 2018 Intel Corporation
  6. #ifndef OPENCV_GAPI_GCOMPUTATION_HPP
  7. #define OPENCV_GAPI_GCOMPUTATION_HPP
  8. #include <functional>
  9. #include <opencv2/gapi/util/util.hpp>
  10. #include <opencv2/gapi/gcommon.hpp>
  11. #include <opencv2/gapi/gproto.hpp>
  12. #include <opencv2/gapi/garg.hpp>
  13. #include <opencv2/gapi/gcompiled.hpp>
  14. #include <opencv2/gapi/gstreaming.hpp>
  15. namespace cv {
  16. namespace detail
  17. {
  18. // FIXME: move to algorithm, cover with separate tests
  19. // FIXME: replace with O(1) version (both memory and compilation time)
  20. template<typename...>
  21. struct last_type;
  22. template<typename T>
  23. struct last_type<T> { using type = T;};
  24. template<typename T, typename... Ts>
  25. struct last_type<T, Ts...> { using type = typename last_type<Ts...>::type; };
  26. template<typename... Ts>
  27. using last_type_t = typename last_type<Ts...>::type;
  28. }
  29. // Forward-declare the serialization objects
  30. namespace gapi {
  31. namespace s11n {
  32. struct IIStream;
  33. struct IOStream;
  34. } // namespace s11n
  35. } // namespace gapi
  36. /**
  37. * \addtogroup gapi_main_classes
  38. * @{
  39. *
  40. * @brief G-API classes for constructed and compiled graphs.
  41. */
  42. /**
  43. * @brief GComputation class represents a captured computation
  44. * graph. GComputation objects form boundaries for expression code
  45. * user writes with G-API, allowing to compile and execute it.
  46. *
  47. * G-API computations are defined with input/output data
  48. * objects. G-API will track automatically which operations connect
  49. * specified outputs to the inputs, forming up a call graph to be
  50. * executed. The below example expresses calculation of Sobel operator
  51. * for edge detection (\f$G = \sqrt{G_x^2 + G_y^2}\f$):
  52. *
  53. * @snippet modules/gapi/samples/api_ref_snippets.cpp graph_def
  54. *
  55. * Full pipeline can be now captured with this object declaration:
  56. *
  57. * @snippet modules/gapi/samples/api_ref_snippets.cpp graph_cap_full
  58. *
  59. * Input/output data objects on which a call graph should be
  60. * reconstructed are passed using special wrappers cv::GIn and
  61. * cv::GOut. G-API will track automatically which operations form a
  62. * path from inputs to outputs and build the execution graph appropriately.
  63. *
  64. * Note that cv::GComputation doesn't take ownership on data objects
  65. * it is defined. Moreover, multiple GComputation objects may be
  66. * defined on the same expressions, e.g. a smaller pipeline which
  67. * expects that image gradients are already pre-calculated may be
  68. * defined like this:
  69. *
  70. * @snippet modules/gapi/samples/api_ref_snippets.cpp graph_cap_sub
  71. *
  72. * The resulting graph would expect two inputs and produce one
  73. * output. In this case, it doesn't matter if gx/gy data objects are
  74. * results of cv::gapi::Sobel operators -- G-API will stop unrolling
  75. * expressions and building the underlying graph one reaching this
  76. * data objects.
  77. *
  78. * The way how GComputation is defined is important as its definition
  79. * specifies graph _protocol_ -- the way how the graph should be
  80. * used. Protocol is defined by number of inputs, number of outputs,
  81. * and shapes of inputs and outputs.
  82. *
  83. * In the above example, sobelEdge expects one Mat on input and
  84. * produces one Mat; while sobelEdgeSub expects two Mats on input and
  85. * produces one Mat. GComputation's protocol defines how other
  86. * computation methods should be used -- cv::GComputation::compile() and
  87. * cv::GComputation::apply(). For example, if a graph is defined on
  88. * two GMat inputs, two cv::Mat objects have to be passed to apply()
  89. * for execution. GComputation checks protocol correctness in runtime
  90. * so passing a different number of objects in apply() or passing
  91. * cv::Scalar instead of cv::Mat there would compile well as a C++
  92. * source but raise an exception in run-time. G-API also comes with a
  93. * typed wrapper cv::GComputationT<> which introduces this type-checking in
  94. * compile-time.
  95. *
  96. * cv::GComputation itself is a thin object which just captures what
  97. * the graph is. The compiled graph (which actually process data) is
  98. * represented by class GCompiled. Use compile() method to generate a
  99. * compiled graph with given compile options. cv::GComputation can
  100. * also be used to process data with implicit graph compilation
  101. * on-the-fly, see apply() for details.
  102. *
  103. * GComputation is a reference-counted object -- once defined, all its
  104. * copies will refer to the same instance.
  105. *
  106. * @sa GCompiled
  107. */
  108. class GAPI_EXPORTS_W GComputation
  109. {
  110. public:
  111. class Priv;
  112. typedef std::function<GComputation()> Generator;
  113. // Various constructors enable different ways to define a computation: /////
  114. // 1. Generic constructors
  115. /**
  116. * @brief Define a computation using a generator function.
  117. *
  118. * Graph can be defined in-place directly at the moment of its
  119. * construction with a lambda:
  120. *
  121. * @snippet modules/gapi/samples/api_ref_snippets.cpp graph_gen
  122. *
  123. * This may be useful since all temporary objects (cv::GMats) and
  124. * namespaces can be localized to scope of lambda, without
  125. * contaminating the parent scope with probably unnecessary objects
  126. * and information.
  127. *
  128. * @param gen generator function which returns a cv::GComputation,
  129. * see Generator.
  130. */
  131. GComputation(const Generator& gen); // Generator
  132. // overload
  133. /**
  134. * @brief Generic GComputation constructor.
  135. *
  136. * Constructs a new graph with a given protocol, specified as a
  137. * flow of operations connecting input/output objects. Throws if
  138. * the passed boundaries are invalid, e.g. if there's no
  139. * functional dependency (path) between given outputs and inputs.
  140. *
  141. * @param ins Input data vector.
  142. * @param outs Output data vector.
  143. *
  144. * @note Don't construct GProtoInputArgs/GProtoOutputArgs objects
  145. * directly, use cv::GIn()/cv::GOut() wrapper functions instead.
  146. *
  147. * @sa @ref gapi_data_objects
  148. */
  149. GAPI_WRAP GComputation(GProtoInputArgs &&ins,
  150. GProtoOutputArgs &&outs); // Arg-to-arg overload
  151. // 2. Syntax sugar and compatibility overloads
  152. /**
  153. * @brief Defines an unary (one input -- one output) computation
  154. *
  155. * @overload
  156. * @param in input GMat of the defined unary computation
  157. * @param out output GMat of the defined unary computation
  158. */
  159. GAPI_WRAP GComputation(GMat in, GMat out); // Unary overload
  160. /**
  161. * @brief Defines an unary (one input -- one output) computation
  162. *
  163. * @overload
  164. * @param in input GMat of the defined unary computation
  165. * @param out output GScalar of the defined unary computation
  166. */
  167. GAPI_WRAP GComputation(GMat in, GScalar out); // Unary overload (scalar)
  168. /**
  169. * @brief Defines a binary (two inputs -- one output) computation
  170. *
  171. * @overload
  172. * @param in1 first input GMat of the defined binary computation
  173. * @param in2 second input GMat of the defined binary computation
  174. * @param out output GMat of the defined binary computation
  175. */
  176. GAPI_WRAP GComputation(GMat in1, GMat in2, GMat out); // Binary overload
  177. /**
  178. * @brief Defines a binary (two inputs -- one output) computation
  179. *
  180. * @overload
  181. * @param in1 first input GMat of the defined binary computation
  182. * @param in2 second input GMat of the defined binary computation
  183. * @param out output GScalar of the defined binary computation
  184. */
  185. GComputation(GMat in1, GMat in2, GScalar out); // Binary
  186. // overload
  187. // (scalar)
  188. /**
  189. * @brief Defines a computation with arbitrary input/output number.
  190. *
  191. * @overload
  192. * @param ins vector of inputs GMats for this computation
  193. * @param outs vector of outputs GMats for this computation
  194. *
  195. * Use this overload for cases when number of computation
  196. * inputs/outputs is not known in compile-time -- e.g. when graph
  197. * is programmatically generated to build an image pyramid with
  198. * the given number of levels, etc.
  199. */
  200. GComputation(const std::vector<GMat> &ins, // Compatibility overload
  201. const std::vector<GMat> &outs);
  202. // Various versions of apply(): ////////////////////////////////////////////
  203. // 1. Generic apply()
  204. /**
  205. * @brief Compile graph on-the-fly and immediately execute it on
  206. * the inputs data vectors.
  207. *
  208. * Number of input/output data objects must match GComputation's
  209. * protocol, also types of host data objects (cv::Mat, cv::Scalar)
  210. * must match the shapes of data objects from protocol (cv::GMat,
  211. * cv::GScalar). If there's a mismatch, a run-time exception will
  212. * be generated.
  213. *
  214. * Internally, a cv::GCompiled object is created for the given
  215. * input format configuration, which then is executed on the input
  216. * data immediately. cv::GComputation caches compiled objects
  217. * produced within apply() -- if this method would be called next
  218. * time with the same input parameters (image formats, image
  219. * resolution, etc), the underlying compiled graph will be reused
  220. * without recompilation. If new metadata doesn't match the cached
  221. * one, the underlying compiled graph is regenerated.
  222. *
  223. * @note compile() always triggers a compilation process and
  224. * produces a new GCompiled object regardless if a similar one has
  225. * been cached via apply() or not.
  226. *
  227. * @param ins vector of input data to process. Don't create
  228. * GRunArgs object manually, use cv::gin() wrapper instead.
  229. * @param outs vector of output data to fill results in. cv::Mat
  230. * objects may be empty in this vector, G-API will automatically
  231. * initialize it with the required format & dimensions. Don't
  232. * create GRunArgsP object manually, use cv::gout() wrapper instead.
  233. * @param args a list of compilation arguments to pass to the
  234. * underlying compilation process. Don't create GCompileArgs
  235. * object manually, use cv::compile_args() wrapper instead.
  236. *
  237. * @sa @ref gapi_data_objects, @ref gapi_compile_args
  238. */
  239. void apply(GRunArgs &&ins, GRunArgsP &&outs, GCompileArgs &&args = {}); // Arg-to-arg overload
  240. /// @private -- Exclude this function from OpenCV documentation
  241. GAPI_WRAP GRunArgs apply(GRunArgs &&ins, GCompileArgs &&args = {});
  242. /// @private -- Exclude this function from OpenCV documentation
  243. void apply(const std::vector<cv::Mat>& ins, // Compatibility overload
  244. const std::vector<cv::Mat>& outs,
  245. GCompileArgs &&args = {});
  246. // 2. Syntax sugar and compatibility overloads
  247. #if !defined(GAPI_STANDALONE)
  248. /**
  249. * @brief Execute an unary computation (with compilation on the fly)
  250. *
  251. * @overload
  252. * @param in input cv::Mat for unary computation
  253. * @param out output cv::Mat for unary computation
  254. * @param args compilation arguments for underlying compilation
  255. * process.
  256. */
  257. void apply(cv::Mat in, cv::Mat &out, GCompileArgs &&args = {}); // Unary overload
  258. /**
  259. * @brief Execute an unary computation (with compilation on the fly)
  260. *
  261. * @overload
  262. * @param in input cv::Mat for unary computation
  263. * @param out output cv::Scalar for unary computation
  264. * @param args compilation arguments for underlying compilation
  265. * process.
  266. */
  267. void apply(cv::Mat in, cv::Scalar &out, GCompileArgs &&args = {}); // Unary overload (scalar)
  268. /**
  269. * @brief Execute a binary computation (with compilation on the fly)
  270. *
  271. * @overload
  272. * @param in1 first input cv::Mat for binary computation
  273. * @param in2 second input cv::Mat for binary computation
  274. * @param out output cv::Mat for binary computation
  275. * @param args compilation arguments for underlying compilation
  276. * process.
  277. */
  278. void apply(cv::Mat in1, cv::Mat in2, cv::Mat &out, GCompileArgs &&args = {}); // Binary overload
  279. /**
  280. * @brief Execute an binary computation (with compilation on the fly)
  281. *
  282. * @overload
  283. * @param in1 first input cv::Mat for binary computation
  284. * @param in2 second input cv::Mat for binary computation
  285. * @param out output cv::Scalar for binary computation
  286. * @param args compilation arguments for underlying compilation
  287. * process.
  288. */
  289. void apply(cv::Mat in1, cv::Mat in2, cv::Scalar &out, GCompileArgs &&args = {}); // Binary overload (scalar)
  290. /**
  291. * @brief Execute a computation with arbitrary number of
  292. * inputs/outputs (with compilation on-the-fly).
  293. *
  294. * @overload
  295. * @param ins vector of input cv::Mat objects to process by the
  296. * computation.
  297. * @param outs vector of output cv::Mat objects to produce by the
  298. * computation.
  299. * @param args compilation arguments for underlying compilation
  300. * process.
  301. *
  302. * Numbers of elements in ins/outs vectors must match numbers of
  303. * inputs/outputs which were used to define this GComputation.
  304. */
  305. void apply(const std::vector<cv::Mat>& ins, // Compatibility overload
  306. std::vector<cv::Mat>& outs,
  307. GCompileArgs &&args = {});
  308. #endif // !defined(GAPI_STANDALONE)
  309. // Various versions of compile(): //////////////////////////////////////////
  310. // 1. Generic compile() - requires metas to be passed as vector
  311. /**
  312. * @brief Compile the computation for specific input format(s).
  313. *
  314. * This method triggers compilation process and produces a new
  315. * GCompiled object which then can process data of the given
  316. * format. Passing data with different format to the compiled
  317. * computation will generate a run-time exception.
  318. *
  319. * @param in_metas vector of input metadata configuration. Grab
  320. * metadata from real data objects (like cv::Mat or cv::Scalar)
  321. * using cv::descr_of(), or create it on your own.
  322. * @param args compilation arguments for this compilation
  323. * process. Compilation arguments directly affect what kind of
  324. * executable object would be produced, e.g. which kernels (and
  325. * thus, devices) would be used to execute computation.
  326. *
  327. * @return GCompiled, an executable computation compiled
  328. * specifically for the given input parameters.
  329. *
  330. * @sa @ref gapi_compile_args
  331. */
  332. GCompiled compile(GMetaArgs &&in_metas, GCompileArgs &&args = {});
  333. // 2. Syntax sugar - variadic list of metas, no extra compile args
  334. // FIXME: SFINAE looks ugly in the generated documentation
  335. /**
  336. * @overload
  337. *
  338. * Takes a variadic parameter pack with metadata
  339. * descriptors for which a compiled object needs to be produced.
  340. *
  341. * @return GCompiled, an executable computation compiled
  342. * specifically for the given input parameters.
  343. */
  344. template<typename... Ts>
  345. auto compile(const Ts&... metas) ->
  346. typename std::enable_if<detail::are_meta_descrs<Ts...>::value, GCompiled>::type
  347. {
  348. return compile(GMetaArgs{GMetaArg(metas)...}, GCompileArgs());
  349. }
  350. // 3. Syntax sugar - variadic list of metas, extra compile args
  351. // (seems optional parameters don't work well when there's an variadic template
  352. // comes first)
  353. //
  354. // Ideally it should look like:
  355. //
  356. // template<typename... Ts>
  357. // GCompiled compile(const Ts&... metas, GCompileArgs &&args)
  358. //
  359. // But not all compilers can handle this (and seems they shouldn't be able to).
  360. // FIXME: SFINAE looks ugly in the generated documentation
  361. /**
  362. * @overload
  363. *
  364. * Takes a variadic parameter pack with metadata
  365. * descriptors for which a compiled object needs to be produced,
  366. * followed by GCompileArgs object representing compilation
  367. * arguments for this process.
  368. *
  369. * @return GCompiled, an executable computation compiled
  370. * specifically for the given input parameters.
  371. */
  372. template<typename... Ts>
  373. auto compile(const Ts&... meta_and_compile_args) ->
  374. typename std::enable_if<detail::are_meta_descrs_but_last<Ts...>::value
  375. && std::is_same<GCompileArgs, detail::last_type_t<Ts...> >::value,
  376. GCompiled>::type
  377. {
  378. //FIXME: wrapping meta_and_compile_args into a tuple to unwrap them inside a helper function is the overkill
  379. return compile(std::make_tuple(meta_and_compile_args...),
  380. typename detail::MkSeq<sizeof...(Ts)-1>::type());
  381. }
  382. // FIXME: Document properly in the Doxygen format
  383. // Video-oriented pipeline compilation:
  384. // 1. A generic version
  385. /**
  386. * @brief Compile the computation for streaming mode.
  387. *
  388. * This method triggers compilation process and produces a new
  389. * GStreamingCompiled object which then can process video stream
  390. * data of the given format. Passing a stream in a different
  391. * format to the compiled computation will generate a run-time
  392. * exception.
  393. *
  394. * @param in_metas vector of input metadata configuration. Grab
  395. * metadata from real data objects (like cv::Mat or cv::Scalar)
  396. * using cv::descr_of(), or create it on your own.
  397. *
  398. * @param args compilation arguments for this compilation
  399. * process. Compilation arguments directly affect what kind of
  400. * executable object would be produced, e.g. which kernels (and
  401. * thus, devices) would be used to execute computation.
  402. *
  403. * @return GStreamingCompiled, a streaming-oriented executable
  404. * computation compiled specifically for the given input
  405. * parameters.
  406. *
  407. * @sa @ref gapi_compile_args
  408. */
  409. GAPI_WRAP GStreamingCompiled compileStreaming(GMetaArgs &&in_metas, GCompileArgs &&args = {});
  410. /**
  411. * @brief Compile the computation for streaming mode.
  412. *
  413. * This method triggers compilation process and produces a new
  414. * GStreamingCompiled object which then can process video stream
  415. * data in any format. Underlying mechanisms will be adjusted to
  416. * every new input video stream automatically, but please note that
  417. * _not all_ existing backends support this (see reshape()).
  418. *
  419. * @param args compilation arguments for this compilation
  420. * process. Compilation arguments directly affect what kind of
  421. * executable object would be produced, e.g. which kernels (and
  422. * thus, devices) would be used to execute computation.
  423. *
  424. * @return GStreamingCompiled, a streaming-oriented executable
  425. * computation compiled for any input image format.
  426. *
  427. * @sa @ref gapi_compile_args
  428. */
  429. GAPI_WRAP GStreamingCompiled compileStreaming(GCompileArgs &&args = {});
  430. // 2. Direct metadata version
  431. /**
  432. * @overload
  433. *
  434. * Takes a variadic parameter pack with metadata
  435. * descriptors for which a compiled object needs to be produced.
  436. *
  437. * @return GStreamingCompiled, a streaming-oriented executable
  438. * computation compiled specifically for the given input
  439. * parameters.
  440. */
  441. template<typename... Ts>
  442. auto compileStreaming(const Ts&... metas) ->
  443. typename std::enable_if<detail::are_meta_descrs<Ts...>::value, GStreamingCompiled>::type
  444. {
  445. return compileStreaming(GMetaArgs{GMetaArg(metas)...}, GCompileArgs());
  446. }
  447. // 2. Direct metadata + compile arguments version
  448. /**
  449. * @overload
  450. *
  451. * Takes a variadic parameter pack with metadata
  452. * descriptors for which a compiled object needs to be produced,
  453. * followed by GCompileArgs object representing compilation
  454. * arguments for this process.
  455. *
  456. * @return GStreamingCompiled, a streaming-oriented executable
  457. * computation compiled specifically for the given input
  458. * parameters.
  459. */
  460. template<typename... Ts>
  461. auto compileStreaming(const Ts&... meta_and_compile_args) ->
  462. typename std::enable_if<detail::are_meta_descrs_but_last<Ts...>::value
  463. && std::is_same<GCompileArgs, detail::last_type_t<Ts...> >::value,
  464. GStreamingCompiled>::type
  465. {
  466. //FIXME: wrapping meta_and_compile_args into a tuple to unwrap them inside a helper function is the overkill
  467. return compileStreaming(std::make_tuple(meta_and_compile_args...),
  468. typename detail::MkSeq<sizeof...(Ts)-1>::type());
  469. }
  470. // Internal use only
  471. /// @private
  472. Priv& priv();
  473. /// @private
  474. const Priv& priv() const;
  475. /// @private
  476. explicit GComputation(cv::gapi::s11n::IIStream &);
  477. /// @private
  478. void serialize(cv::gapi::s11n::IOStream &) const;
  479. protected:
  480. // 4. Helper methods for (3)
  481. /// @private
  482. template<typename... Ts, int... IIs>
  483. GCompiled compile(const std::tuple<Ts...> &meta_and_compile_args, detail::Seq<IIs...>)
  484. {
  485. GMetaArgs meta_args = {GMetaArg(std::get<IIs>(meta_and_compile_args))...};
  486. GCompileArgs comp_args = std::get<sizeof...(Ts)-1>(meta_and_compile_args);
  487. return compile(std::move(meta_args), std::move(comp_args));
  488. }
  489. template<typename... Ts, int... IIs>
  490. GStreamingCompiled compileStreaming(const std::tuple<Ts...> &meta_and_compile_args, detail::Seq<IIs...>)
  491. {
  492. GMetaArgs meta_args = {GMetaArg(std::get<IIs>(meta_and_compile_args))...};
  493. GCompileArgs comp_args = std::get<sizeof...(Ts)-1>(meta_and_compile_args);
  494. return compileStreaming(std::move(meta_args), std::move(comp_args));
  495. }
  496. void recompile(GMetaArgs&& in_metas, GCompileArgs &&args);
  497. /// @private
  498. std::shared_ptr<Priv> m_priv;
  499. };
  500. /** @} */
  501. namespace gapi
  502. {
  503. // FIXME: all these standalone functions need to be added to some
  504. // common documentation section
  505. /**
  506. * @brief Define an tagged island (subgraph) within a computation.
  507. *
  508. * Declare an Island tagged with `name` and defined from `ins` to `outs`
  509. * (exclusively, as ins/outs are data objects, and regioning is done on
  510. * operations level).
  511. * Throws if any operation between `ins` and `outs` are already assigned
  512. * to another island.
  513. *
  514. * Islands allow to partition graph into subgraphs, fine-tuning
  515. * the way it is scheduled by the underlying executor.
  516. *
  517. * @param name name of the Island to create
  518. * @param ins vector of input data objects where the subgraph
  519. * begins
  520. * @param outs vector of output data objects where the subgraph
  521. * ends.
  522. *
  523. * The way how an island is defined is similar to how
  524. * cv::GComputation is defined on input/output data objects.
  525. * Same rules apply here as well -- if there's no functional
  526. * dependency between inputs and outputs or there's not enough
  527. * input data objects were specified to properly calculate all
  528. * outputs, an exception is thrown.
  529. *
  530. * Use cv::GIn() / cv::GOut() to specify input/output vectors.
  531. */
  532. void GAPI_EXPORTS island(const std::string &name,
  533. GProtoInputArgs &&ins,
  534. GProtoOutputArgs &&outs);
  535. } // namespace gapi
  536. } // namespace cv
  537. #endif // OPENCV_GAPI_GCOMPUTATION_HPP