infer.hpp 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492
  1. // This file is part of OpenCV project.
  2. // It is subject to the license terms in the LICENSE file found in the top-level directory
  3. // of this distribution and at http://opencv.org/license.html.
  4. //
  5. // Copyright (C) 2019-2020 Intel Corporation
  6. #ifndef OPENCV_GAPI_INFER_HPP
  7. #define OPENCV_GAPI_INFER_HPP
  8. // FIXME: Inference API is currently only available in full mode
  9. #if !defined(GAPI_STANDALONE)
  10. #include <functional>
  11. #include <string> // string
  12. #include <utility> // tuple
  13. #include <type_traits> // is_same, false_type
  14. #include <opencv2/gapi/util/util.hpp> // all_satisfy
  15. #include <opencv2/gapi/util/any.hpp> // any<>
  16. #include <opencv2/gapi/gkernel.hpp> // GKernelType[M], GBackend
  17. #include <opencv2/gapi/garg.hpp> // GArg
  18. #include <opencv2/gapi/gcommon.hpp> // CompileArgTag
  19. #include <opencv2/gapi/gmetaarg.hpp> // GMetaArg
  20. namespace cv {
  21. template<typename, typename> class GNetworkType;
  22. namespace detail {
  23. // Infer ///////////////////////////////////////////////////////////////////////
  24. template<typename T>
  25. struct accepted_infer_types {
  26. static constexpr const auto value =
  27. std::is_same<typename std::decay<T>::type, cv::GMat>::value
  28. || std::is_same<typename std::decay<T>::type, cv::GFrame>::value;
  29. };
  30. template<typename... Ts>
  31. using valid_infer_types = all_satisfy<accepted_infer_types, Ts...>;
  32. // Infer2 //////////////////////////////////////////////////////////////////////
  33. template<typename, typename>
  34. struct valid_infer2_types;
  35. // Terminal case 1 (50/50 success)
  36. template<typename T>
  37. struct valid_infer2_types< std::tuple<cv::GMat>, std::tuple<T> > {
  38. // By default, Nets are limited to GMat argument types only
  39. // for infer2, every GMat argument may translate to either
  40. // GArray<GMat> or GArray<Rect>. GArray<> part is stripped
  41. // already at this point.
  42. static constexpr const auto value =
  43. std::is_same<typename std::decay<T>::type, cv::GMat>::value
  44. || std::is_same<typename std::decay<T>::type, cv::Rect>::value;
  45. };
  46. // Terminal case 2 (100% failure)
  47. template<typename... Ts>
  48. struct valid_infer2_types< std::tuple<>, std::tuple<Ts...> >
  49. : public std::false_type {
  50. };
  51. // Terminal case 3 (100% failure)
  52. template<typename... Ns>
  53. struct valid_infer2_types< std::tuple<Ns...>, std::tuple<> >
  54. : public std::false_type {
  55. };
  56. // Recursion -- generic
  57. template<typename... Ns, typename T, typename...Ts>
  58. struct valid_infer2_types< std::tuple<cv::GMat,Ns...>, std::tuple<T,Ts...> > {
  59. static constexpr const auto value =
  60. valid_infer2_types< std::tuple<cv::GMat>, std::tuple<T> >::value
  61. && valid_infer2_types< std::tuple<Ns...>, std::tuple<Ts...> >::value;
  62. };
  63. } // namespace detail
  64. // TODO: maybe tuple_wrap_helper from util.hpp may help with this.
  65. // Multiple-return-value network definition (specialized base class)
  66. template<typename K, typename... R, typename... Args>
  67. class GNetworkType<K, std::function<std::tuple<R...>(Args...)> >
  68. {
  69. public:
  70. using InArgs = std::tuple<Args...>;
  71. using OutArgs = std::tuple<R...>;
  72. using Result = OutArgs;
  73. using API = std::function<Result(Args...)>;
  74. using ResultL = std::tuple< cv::GArray<R>... >;
  75. };
  76. // Single-return-value network definition (specialized base class)
  77. template<typename K, typename R, typename... Args>
  78. class GNetworkType<K, std::function<R(Args...)> >
  79. {
  80. public:
  81. using InArgs = std::tuple<Args...>;
  82. using OutArgs = std::tuple<R>;
  83. using Result = R;
  84. using API = std::function<R(Args...)>;
  85. using ResultL = cv::GArray<R>;
  86. };
  87. // InferAPI: Accepts either GMat or GFrame for very individual network's input
  88. template<class Net, class... Ts>
  89. struct InferAPI {
  90. using type = typename std::enable_if
  91. < detail::valid_infer_types<Ts...>::value
  92. && std::tuple_size<typename Net::InArgs>::value == sizeof...(Ts)
  93. , std::function<typename Net::Result(Ts...)>
  94. >::type;
  95. };
  96. // InferAPIRoi: Accepts a rectangle and either GMat or GFrame
  97. template<class Net, class T>
  98. struct InferAPIRoi {
  99. using type = typename std::enable_if
  100. < detail::valid_infer_types<T>::value
  101. && std::tuple_size<typename Net::InArgs>::value == 1u
  102. , std::function<typename Net::Result(cv::GOpaque<cv::Rect>, T)>
  103. >::type;
  104. };
  105. // InferAPIList: Accepts a list of rectangles and list of GMat/GFrames;
  106. // crops every input.
  107. template<class Net, class... Ts>
  108. struct InferAPIList {
  109. using type = typename std::enable_if
  110. < detail::valid_infer_types<Ts...>::value
  111. && std::tuple_size<typename Net::InArgs>::value == sizeof...(Ts)
  112. , std::function<typename Net::ResultL(cv::GArray<cv::Rect>, Ts...)>
  113. >::type;
  114. };
  115. // APIList2 is also template to allow different calling options
  116. // (GArray<cv::Rect> vs GArray<cv::GMat> per input)
  117. template<class Net, typename T, class... Ts>
  118. struct InferAPIList2 {
  119. using type = typename std::enable_if
  120. < detail::valid_infer_types<T>::value &&
  121. cv::detail::valid_infer2_types< typename Net::InArgs
  122. , std::tuple<Ts...> >::value,
  123. std::function<typename Net::ResultL(T, cv::GArray<Ts>...)>
  124. >::type;
  125. };
  126. // Base "Infer" kernel. Note - for whatever network, kernel ID
  127. // is always the same. Different inference calls are distinguished by
  128. // network _tag_ (an extra field in GCall)
  129. //
  130. // getOutMeta is a stub callback collected by G-API kernel subsystem
  131. // automatically. This is a rare case when this callback is defined by
  132. // a particular backend, not by a network itself.
  133. struct GInferBase {
  134. static constexpr const char * id() {
  135. return "org.opencv.dnn.infer"; // Universal stub
  136. }
  137. static GMetaArgs getOutMeta(const GMetaArgs &, const GArgs &) {
  138. return GMetaArgs{}; // One more universal stub
  139. }
  140. };
  141. // Struct stores network input/output names.
  142. // Used by infer<Generic>
  143. struct InOutInfo
  144. {
  145. std::vector<std::string> in_names;
  146. std::vector<std::string> out_names;
  147. };
  148. /**
  149. * @{
  150. * @brief G-API object used to collect network inputs
  151. */
  152. class GAPI_EXPORTS_W_SIMPLE GInferInputs
  153. {
  154. using Map = std::unordered_map<std::string, GMat>;
  155. public:
  156. GAPI_WRAP GInferInputs();
  157. GAPI_WRAP void setInput(const std::string& name, const cv::GMat& value);
  158. cv::GMat& operator[](const std::string& name);
  159. const Map& getBlobs() const;
  160. private:
  161. std::shared_ptr<Map> in_blobs;
  162. };
  163. /** @} */
  164. /**
  165. * @{
  166. * @brief G-API object used to collect network outputs
  167. */
  168. struct GAPI_EXPORTS_W_SIMPLE GInferOutputs
  169. {
  170. public:
  171. GAPI_WRAP GInferOutputs() = default;
  172. GInferOutputs(std::shared_ptr<cv::GCall> call);
  173. GAPI_WRAP cv::GMat at(const std::string& name);
  174. private:
  175. struct Priv;
  176. std::shared_ptr<Priv> m_priv;
  177. };
  178. /** @} */
  179. // Base "InferROI" kernel.
  180. // All notes from "Infer" kernel apply here as well.
  181. struct GInferROIBase {
  182. static constexpr const char * id() {
  183. return "org.opencv.dnn.infer-roi"; // Universal stub
  184. }
  185. static GMetaArgs getOutMeta(const GMetaArgs &, const GArgs &) {
  186. return GMetaArgs{}; // One more universal stub
  187. }
  188. };
  189. // Base "Infer list" kernel.
  190. // All notes from "Infer" kernel apply here as well.
  191. struct GInferListBase {
  192. static constexpr const char * id() {
  193. return "org.opencv.dnn.infer-roi-list-1"; // Universal stub
  194. }
  195. static GMetaArgs getOutMeta(const GMetaArgs &, const GArgs &) {
  196. return GMetaArgs{}; // One more universal stub
  197. }
  198. };
  199. // Base "Infer list 2" kernel.
  200. // All notes from "Infer" kernel apply here as well.
  201. struct GInferList2Base {
  202. static constexpr const char * id() {
  203. return "org.opencv.dnn.infer-roi-list-2"; // Universal stub
  204. }
  205. static GMetaArgs getOutMeta(const GMetaArgs &, const GArgs &) {
  206. return GMetaArgs{}; // One more universal stub
  207. }
  208. };
  209. // A generic inference kernel. API (::on()) is fully defined by the Net
  210. // template parameter.
  211. // Acts as a regular kernel in graph (via KernelTypeMedium).
  212. template<typename Net, typename... Args>
  213. struct GInfer final
  214. : public GInferBase
  215. , public detail::KernelTypeMedium< GInfer<Net, Args...>
  216. , typename InferAPI<Net, Args...>::type > {
  217. using GInferBase::getOutMeta; // FIXME: name lookup conflict workaround?
  218. static constexpr const char* tag() { return Net::tag(); }
  219. };
  220. // A specific roi-inference kernel. API (::on()) is fixed here and
  221. // verified against Net.
  222. template<typename Net, typename T>
  223. struct GInferROI final
  224. : public GInferROIBase
  225. , public detail::KernelTypeMedium< GInferROI<Net, T>
  226. , typename InferAPIRoi<Net, T>::type > {
  227. using GInferROIBase::getOutMeta; // FIXME: name lookup conflict workaround?
  228. static constexpr const char* tag() { return Net::tag(); }
  229. };
  230. // A generic roi-list inference kernel. API (::on()) is derived from
  231. // the Net template parameter (see more in infer<> overload).
  232. template<typename Net, typename... Args>
  233. struct GInferList final
  234. : public GInferListBase
  235. , public detail::KernelTypeMedium< GInferList<Net, Args...>
  236. , typename InferAPIList<Net, Args...>::type > {
  237. using GInferListBase::getOutMeta; // FIXME: name lookup conflict workaround?
  238. static constexpr const char* tag() { return Net::tag(); }
  239. };
  240. // An even more generic roi-list inference kernel. API (::on()) is
  241. // derived from the Net template parameter (see more in infer<>
  242. // overload).
  243. // Takes an extra variadic template list to reflect how this network
  244. // was called (with Rects or GMats as array parameters)
  245. template<typename Net, typename T, typename... Args>
  246. struct GInferList2 final
  247. : public GInferList2Base
  248. , public detail::KernelTypeMedium< GInferList2<Net, T, Args...>
  249. , typename InferAPIList2<Net, T, Args...>::type > {
  250. using GInferList2Base::getOutMeta; // FIXME: name lookup conflict workaround?
  251. static constexpr const char* tag() { return Net::tag(); }
  252. };
  253. } // namespace cv
  254. // FIXME: Probably the <API> signature makes a function/tuple/function round-trip
  255. #define G_API_NET(Class, API, Tag) \
  256. struct Class final: public cv::GNetworkType<Class, std::function API> { \
  257. static constexpr const char * tag() { return Tag; } \
  258. }
  259. namespace cv {
  260. namespace gapi {
  261. /** @brief Calculates response for the specified network (template
  262. * parameter) for the specified region in the source image.
  263. * Currently expects a single-input network only.
  264. *
  265. * @tparam A network type defined with G_API_NET() macro.
  266. * @param in input image where to take ROI from.
  267. * @param roi an object describing the region of interest
  268. * in the source image. May be calculated in the same graph dynamically.
  269. * @return an object of return type as defined in G_API_NET().
  270. * If a network has multiple return values (defined with a tuple), a tuple of
  271. * objects of appropriate type is returned.
  272. * @sa G_API_NET()
  273. */
  274. template<typename Net, typename T>
  275. typename Net::Result infer(cv::GOpaque<cv::Rect> roi, T in) {
  276. return GInferROI<Net, T>::on(roi, in);
  277. }
  278. /** @brief Calculates responses for the specified network (template
  279. * parameter) for every region in the source image.
  280. *
  281. * @tparam A network type defined with G_API_NET() macro.
  282. * @param roi a list of rectangles describing regions of interest
  283. * in the source image. Usually an output of object detector or tracker.
  284. * @param args network's input parameters as specified in G_API_NET() macro.
  285. * NOTE: verified to work reliably with 1-input topologies only.
  286. * @return a list of objects of return type as defined in G_API_NET().
  287. * If a network has multiple return values (defined with a tuple), a tuple of
  288. * GArray<> objects is returned with the appropriate types inside.
  289. * @sa G_API_NET()
  290. */
  291. template<typename Net, typename... Args>
  292. typename Net::ResultL infer(cv::GArray<cv::Rect> roi, Args&&... args) {
  293. return GInferList<Net, Args...>::on(roi, std::forward<Args>(args)...);
  294. }
  295. /** @brief Calculates responses for the specified network (template
  296. * parameter) for every region in the source image, extended version.
  297. *
  298. * @tparam A network type defined with G_API_NET() macro.
  299. * @param image A source image containing regions of interest
  300. * @param args GArray<> objects of cv::Rect or cv::GMat, one per every
  301. * network input:
  302. * - If a cv::GArray<cv::Rect> is passed, the appropriate
  303. * regions are taken from `image` and preprocessed to this particular
  304. * network input;
  305. * - If a cv::GArray<cv::GMat> is passed, the underlying data traited
  306. * as tensor (no automatic preprocessing happen).
  307. * @return a list of objects of return type as defined in G_API_NET().
  308. * If a network has multiple return values (defined with a tuple), a tuple of
  309. * GArray<> objects is returned with the appropriate types inside.
  310. * @sa G_API_NET()
  311. */
  312. template<typename Net, typename T, typename... Args>
  313. typename Net::ResultL infer2(T image, cv::GArray<Args>... args) {
  314. // FIXME: Declared as "2" because in the current form it steals
  315. // overloads from the regular infer
  316. return GInferList2<Net, T, Args...>::on(image, args...);
  317. }
  318. /**
  319. * @brief Calculates response for the specified network (template
  320. * parameter) given the input data.
  321. *
  322. * @tparam A network type defined with G_API_NET() macro.
  323. * @param args network's input parameters as specified in G_API_NET() macro.
  324. * @return an object of return type as defined in G_API_NET().
  325. * If a network has multiple return values (defined with a tuple), a tuple of
  326. * objects of appropriate type is returned.
  327. * @sa G_API_NET()
  328. */
  329. template<typename Net, typename... Args>
  330. typename Net::Result infer(Args&&... args) {
  331. return GInfer<Net, Args...>::on(std::forward<Args>(args)...);
  332. }
  333. /**
  334. * @brief Special network type
  335. */
  336. struct Generic { };
  337. /**
  338. * @brief Calculates response for generic network
  339. *
  340. * @param tag a network tag
  341. * @param inputs networks's inputs
  342. * @return a GInferOutputs
  343. */
  344. template<typename T = Generic> GInferOutputs
  345. infer(const std::string& tag, const GInferInputs& inputs)
  346. {
  347. std::vector<GArg> input_args;
  348. std::vector<std::string> input_names;
  349. const auto& blobs = inputs.getBlobs();
  350. for (auto&& p : blobs)
  351. {
  352. input_names.push_back(p.first);
  353. input_args.emplace_back(p.second);
  354. }
  355. GKinds kinds(blobs.size(), cv::detail::OpaqueKind::CV_MAT);
  356. auto call = std::make_shared<cv::GCall>(GKernel{
  357. GInferBase::id(),
  358. tag,
  359. GInferBase::getOutMeta,
  360. {}, // outShape will be filled later
  361. std::move(kinds),
  362. {}, // outCtors will be filled later
  363. });
  364. call->setArgs(std::move(input_args));
  365. call->params() = InOutInfo{input_names, {}};
  366. return GInferOutputs{std::move(call)};
  367. }
  368. GAPI_EXPORTS_W inline GInferOutputs infer(const String& name, const GInferInputs& inputs)
  369. {
  370. return infer<Generic>(name, inputs);
  371. }
  372. } // namespace gapi
  373. } // namespace cv
  374. #endif // GAPI_STANDALONE
  375. namespace cv {
  376. namespace gapi {
  377. // Note: the below code _is_ part of STANDALONE build,
  378. // just to make our compiler code compileable.
  379. // A type-erased form of network parameters.
  380. // Similar to how a type-erased GKernel is represented and used.
  381. struct GAPI_EXPORTS GNetParam {
  382. std::string tag; // FIXME: const?
  383. GBackend backend; // Specifies the execution model
  384. util::any params; // Backend-interpreted parameter structure
  385. };
  386. /** \addtogroup gapi_compile_args
  387. * @{
  388. */
  389. /**
  390. * @brief A container class for network configurations. Similar to
  391. * GKernelPackage.Use cv::gapi::networks() to construct this object.
  392. *
  393. * @sa cv::gapi::networks
  394. */
  395. struct GAPI_EXPORTS_W_SIMPLE GNetPackage {
  396. GAPI_WRAP GNetPackage() = default;
  397. explicit GNetPackage(std::initializer_list<GNetParam> ii);
  398. std::vector<GBackend> backends() const;
  399. std::vector<GNetParam> networks;
  400. };
  401. /** @} gapi_compile_args */
  402. } // namespace gapi
  403. namespace detail {
  404. template<typename T>
  405. gapi::GNetParam strip(T&& t) {
  406. return gapi::GNetParam { t.tag()
  407. , t.backend()
  408. , t.params()
  409. };
  410. }
  411. template<> struct CompileArgTag<cv::gapi::GNetPackage> {
  412. static const char* tag() { return "gapi.net_package"; }
  413. };
  414. } // namespace cv::detail
  415. namespace gapi {
  416. template<typename... Args>
  417. cv::gapi::GNetPackage networks(Args&&... args) {
  418. return cv::gapi::GNetPackage({ cv::detail::strip(args)... });
  419. }
  420. } // namespace gapi
  421. } // namespace cv
  422. #endif // OPENCV_GAPI_INFER_HPP