avx512vldqintrin.h 47 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185
  1. /*===---- avx512vldqintrin.h - AVX512VL and AVX512DQ intrinsics ------------===
  2. *
  3. * Permission is hereby granted, free of charge, to any person obtaining a copy
  4. * of this software and associated documentation files (the "Software"), to deal
  5. * in the Software without restriction, including without limitation the rights
  6. * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  7. * copies of the Software, and to permit persons to whom the Software is
  8. * furnished to do so, subject to the following conditions:
  9. *
  10. * The above copyright notice and this permission notice shall be included in
  11. * all copies or substantial portions of the Software.
  12. *
  13. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  16. * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  17. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  18. * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  19. * THE SOFTWARE.
  20. *
  21. *===-----------------------------------------------------------------------===
  22. */
  23. #ifndef __IMMINTRIN_H
  24. #error "Never use <avx512vldqintrin.h> directly; include <immintrin.h> instead."
  25. #endif
  26. #ifndef __AVX512VLDQINTRIN_H
  27. #define __AVX512VLDQINTRIN_H
  28. /* Define the default attributes for the functions in this file. */
  29. #define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("avx512vl,avx512dq"), __min_vector_width__(128)))
  30. #define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("avx512vl,avx512dq"), __min_vector_width__(256)))
  31. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  32. _mm256_mullo_epi64 (__m256i __A, __m256i __B) {
  33. return (__m256i) ((__v4du) __A * (__v4du) __B);
  34. }
  35. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  36. _mm256_mask_mullo_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
  37. return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
  38. (__v4di)_mm256_mullo_epi64(__A, __B),
  39. (__v4di)__W);
  40. }
  41. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  42. _mm256_maskz_mullo_epi64(__mmask8 __U, __m256i __A, __m256i __B) {
  43. return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
  44. (__v4di)_mm256_mullo_epi64(__A, __B),
  45. (__v4di)_mm256_setzero_si256());
  46. }
  47. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  48. _mm_mullo_epi64 (__m128i __A, __m128i __B) {
  49. return (__m128i) ((__v2du) __A * (__v2du) __B);
  50. }
  51. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  52. _mm_mask_mullo_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
  53. return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
  54. (__v2di)_mm_mullo_epi64(__A, __B),
  55. (__v2di)__W);
  56. }
  57. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  58. _mm_maskz_mullo_epi64(__mmask8 __U, __m128i __A, __m128i __B) {
  59. return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
  60. (__v2di)_mm_mullo_epi64(__A, __B),
  61. (__v2di)_mm_setzero_si128());
  62. }
  63. static __inline__ __m256d __DEFAULT_FN_ATTRS256
  64. _mm256_mask_andnot_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
  65. return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
  66. (__v4df)_mm256_andnot_pd(__A, __B),
  67. (__v4df)__W);
  68. }
  69. static __inline__ __m256d __DEFAULT_FN_ATTRS256
  70. _mm256_maskz_andnot_pd(__mmask8 __U, __m256d __A, __m256d __B) {
  71. return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
  72. (__v4df)_mm256_andnot_pd(__A, __B),
  73. (__v4df)_mm256_setzero_pd());
  74. }
  75. static __inline__ __m128d __DEFAULT_FN_ATTRS128
  76. _mm_mask_andnot_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
  77. return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
  78. (__v2df)_mm_andnot_pd(__A, __B),
  79. (__v2df)__W);
  80. }
  81. static __inline__ __m128d __DEFAULT_FN_ATTRS128
  82. _mm_maskz_andnot_pd(__mmask8 __U, __m128d __A, __m128d __B) {
  83. return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
  84. (__v2df)_mm_andnot_pd(__A, __B),
  85. (__v2df)_mm_setzero_pd());
  86. }
  87. static __inline__ __m256 __DEFAULT_FN_ATTRS256
  88. _mm256_mask_andnot_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
  89. return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
  90. (__v8sf)_mm256_andnot_ps(__A, __B),
  91. (__v8sf)__W);
  92. }
  93. static __inline__ __m256 __DEFAULT_FN_ATTRS256
  94. _mm256_maskz_andnot_ps(__mmask8 __U, __m256 __A, __m256 __B) {
  95. return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
  96. (__v8sf)_mm256_andnot_ps(__A, __B),
  97. (__v8sf)_mm256_setzero_ps());
  98. }
  99. static __inline__ __m128 __DEFAULT_FN_ATTRS128
  100. _mm_mask_andnot_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
  101. return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
  102. (__v4sf)_mm_andnot_ps(__A, __B),
  103. (__v4sf)__W);
  104. }
  105. static __inline__ __m128 __DEFAULT_FN_ATTRS128
  106. _mm_maskz_andnot_ps(__mmask8 __U, __m128 __A, __m128 __B) {
  107. return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
  108. (__v4sf)_mm_andnot_ps(__A, __B),
  109. (__v4sf)_mm_setzero_ps());
  110. }
  111. static __inline__ __m256d __DEFAULT_FN_ATTRS256
  112. _mm256_mask_and_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
  113. return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
  114. (__v4df)_mm256_and_pd(__A, __B),
  115. (__v4df)__W);
  116. }
  117. static __inline__ __m256d __DEFAULT_FN_ATTRS256
  118. _mm256_maskz_and_pd(__mmask8 __U, __m256d __A, __m256d __B) {
  119. return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
  120. (__v4df)_mm256_and_pd(__A, __B),
  121. (__v4df)_mm256_setzero_pd());
  122. }
  123. static __inline__ __m128d __DEFAULT_FN_ATTRS128
  124. _mm_mask_and_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
  125. return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
  126. (__v2df)_mm_and_pd(__A, __B),
  127. (__v2df)__W);
  128. }
  129. static __inline__ __m128d __DEFAULT_FN_ATTRS128
  130. _mm_maskz_and_pd(__mmask8 __U, __m128d __A, __m128d __B) {
  131. return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
  132. (__v2df)_mm_and_pd(__A, __B),
  133. (__v2df)_mm_setzero_pd());
  134. }
  135. static __inline__ __m256 __DEFAULT_FN_ATTRS256
  136. _mm256_mask_and_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
  137. return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
  138. (__v8sf)_mm256_and_ps(__A, __B),
  139. (__v8sf)__W);
  140. }
  141. static __inline__ __m256 __DEFAULT_FN_ATTRS256
  142. _mm256_maskz_and_ps(__mmask8 __U, __m256 __A, __m256 __B) {
  143. return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
  144. (__v8sf)_mm256_and_ps(__A, __B),
  145. (__v8sf)_mm256_setzero_ps());
  146. }
  147. static __inline__ __m128 __DEFAULT_FN_ATTRS128
  148. _mm_mask_and_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
  149. return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
  150. (__v4sf)_mm_and_ps(__A, __B),
  151. (__v4sf)__W);
  152. }
  153. static __inline__ __m128 __DEFAULT_FN_ATTRS128
  154. _mm_maskz_and_ps(__mmask8 __U, __m128 __A, __m128 __B) {
  155. return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
  156. (__v4sf)_mm_and_ps(__A, __B),
  157. (__v4sf)_mm_setzero_ps());
  158. }
  159. static __inline__ __m256d __DEFAULT_FN_ATTRS256
  160. _mm256_mask_xor_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
  161. return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
  162. (__v4df)_mm256_xor_pd(__A, __B),
  163. (__v4df)__W);
  164. }
  165. static __inline__ __m256d __DEFAULT_FN_ATTRS256
  166. _mm256_maskz_xor_pd(__mmask8 __U, __m256d __A, __m256d __B) {
  167. return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
  168. (__v4df)_mm256_xor_pd(__A, __B),
  169. (__v4df)_mm256_setzero_pd());
  170. }
  171. static __inline__ __m128d __DEFAULT_FN_ATTRS128
  172. _mm_mask_xor_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
  173. return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
  174. (__v2df)_mm_xor_pd(__A, __B),
  175. (__v2df)__W);
  176. }
  177. static __inline__ __m128d __DEFAULT_FN_ATTRS128
  178. _mm_maskz_xor_pd (__mmask8 __U, __m128d __A, __m128d __B) {
  179. return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
  180. (__v2df)_mm_xor_pd(__A, __B),
  181. (__v2df)_mm_setzero_pd());
  182. }
  183. static __inline__ __m256 __DEFAULT_FN_ATTRS256
  184. _mm256_mask_xor_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
  185. return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
  186. (__v8sf)_mm256_xor_ps(__A, __B),
  187. (__v8sf)__W);
  188. }
  189. static __inline__ __m256 __DEFAULT_FN_ATTRS256
  190. _mm256_maskz_xor_ps(__mmask8 __U, __m256 __A, __m256 __B) {
  191. return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
  192. (__v8sf)_mm256_xor_ps(__A, __B),
  193. (__v8sf)_mm256_setzero_ps());
  194. }
  195. static __inline__ __m128 __DEFAULT_FN_ATTRS128
  196. _mm_mask_xor_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
  197. return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
  198. (__v4sf)_mm_xor_ps(__A, __B),
  199. (__v4sf)__W);
  200. }
  201. static __inline__ __m128 __DEFAULT_FN_ATTRS128
  202. _mm_maskz_xor_ps(__mmask8 __U, __m128 __A, __m128 __B) {
  203. return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
  204. (__v4sf)_mm_xor_ps(__A, __B),
  205. (__v4sf)_mm_setzero_ps());
  206. }
  207. static __inline__ __m256d __DEFAULT_FN_ATTRS256
  208. _mm256_mask_or_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
  209. return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
  210. (__v4df)_mm256_or_pd(__A, __B),
  211. (__v4df)__W);
  212. }
  213. static __inline__ __m256d __DEFAULT_FN_ATTRS256
  214. _mm256_maskz_or_pd(__mmask8 __U, __m256d __A, __m256d __B) {
  215. return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
  216. (__v4df)_mm256_or_pd(__A, __B),
  217. (__v4df)_mm256_setzero_pd());
  218. }
  219. static __inline__ __m128d __DEFAULT_FN_ATTRS128
  220. _mm_mask_or_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
  221. return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
  222. (__v2df)_mm_or_pd(__A, __B),
  223. (__v2df)__W);
  224. }
  225. static __inline__ __m128d __DEFAULT_FN_ATTRS128
  226. _mm_maskz_or_pd(__mmask8 __U, __m128d __A, __m128d __B) {
  227. return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
  228. (__v2df)_mm_or_pd(__A, __B),
  229. (__v2df)_mm_setzero_pd());
  230. }
  231. static __inline__ __m256 __DEFAULT_FN_ATTRS256
  232. _mm256_mask_or_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
  233. return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
  234. (__v8sf)_mm256_or_ps(__A, __B),
  235. (__v8sf)__W);
  236. }
  237. static __inline__ __m256 __DEFAULT_FN_ATTRS256
  238. _mm256_maskz_or_ps(__mmask8 __U, __m256 __A, __m256 __B) {
  239. return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
  240. (__v8sf)_mm256_or_ps(__A, __B),
  241. (__v8sf)_mm256_setzero_ps());
  242. }
  243. static __inline__ __m128 __DEFAULT_FN_ATTRS128
  244. _mm_mask_or_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
  245. return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
  246. (__v4sf)_mm_or_ps(__A, __B),
  247. (__v4sf)__W);
  248. }
  249. static __inline__ __m128 __DEFAULT_FN_ATTRS128
  250. _mm_maskz_or_ps(__mmask8 __U, __m128 __A, __m128 __B) {
  251. return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
  252. (__v4sf)_mm_or_ps(__A, __B),
  253. (__v4sf)_mm_setzero_ps());
  254. }
  255. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  256. _mm_cvtpd_epi64 (__m128d __A) {
  257. return (__m128i) __builtin_ia32_cvtpd2qq128_mask ((__v2df) __A,
  258. (__v2di) _mm_setzero_si128(),
  259. (__mmask8) -1);
  260. }
  261. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  262. _mm_mask_cvtpd_epi64 (__m128i __W, __mmask8 __U, __m128d __A) {
  263. return (__m128i) __builtin_ia32_cvtpd2qq128_mask ((__v2df) __A,
  264. (__v2di) __W,
  265. (__mmask8) __U);
  266. }
  267. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  268. _mm_maskz_cvtpd_epi64 (__mmask8 __U, __m128d __A) {
  269. return (__m128i) __builtin_ia32_cvtpd2qq128_mask ((__v2df) __A,
  270. (__v2di) _mm_setzero_si128(),
  271. (__mmask8) __U);
  272. }
  273. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  274. _mm256_cvtpd_epi64 (__m256d __A) {
  275. return (__m256i) __builtin_ia32_cvtpd2qq256_mask ((__v4df) __A,
  276. (__v4di) _mm256_setzero_si256(),
  277. (__mmask8) -1);
  278. }
  279. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  280. _mm256_mask_cvtpd_epi64 (__m256i __W, __mmask8 __U, __m256d __A) {
  281. return (__m256i) __builtin_ia32_cvtpd2qq256_mask ((__v4df) __A,
  282. (__v4di) __W,
  283. (__mmask8) __U);
  284. }
  285. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  286. _mm256_maskz_cvtpd_epi64 (__mmask8 __U, __m256d __A) {
  287. return (__m256i) __builtin_ia32_cvtpd2qq256_mask ((__v4df) __A,
  288. (__v4di) _mm256_setzero_si256(),
  289. (__mmask8) __U);
  290. }
  291. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  292. _mm_cvtpd_epu64 (__m128d __A) {
  293. return (__m128i) __builtin_ia32_cvtpd2uqq128_mask ((__v2df) __A,
  294. (__v2di) _mm_setzero_si128(),
  295. (__mmask8) -1);
  296. }
  297. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  298. _mm_mask_cvtpd_epu64 (__m128i __W, __mmask8 __U, __m128d __A) {
  299. return (__m128i) __builtin_ia32_cvtpd2uqq128_mask ((__v2df) __A,
  300. (__v2di) __W,
  301. (__mmask8) __U);
  302. }
  303. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  304. _mm_maskz_cvtpd_epu64 (__mmask8 __U, __m128d __A) {
  305. return (__m128i) __builtin_ia32_cvtpd2uqq128_mask ((__v2df) __A,
  306. (__v2di) _mm_setzero_si128(),
  307. (__mmask8) __U);
  308. }
  309. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  310. _mm256_cvtpd_epu64 (__m256d __A) {
  311. return (__m256i) __builtin_ia32_cvtpd2uqq256_mask ((__v4df) __A,
  312. (__v4di) _mm256_setzero_si256(),
  313. (__mmask8) -1);
  314. }
  315. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  316. _mm256_mask_cvtpd_epu64 (__m256i __W, __mmask8 __U, __m256d __A) {
  317. return (__m256i) __builtin_ia32_cvtpd2uqq256_mask ((__v4df) __A,
  318. (__v4di) __W,
  319. (__mmask8) __U);
  320. }
  321. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  322. _mm256_maskz_cvtpd_epu64 (__mmask8 __U, __m256d __A) {
  323. return (__m256i) __builtin_ia32_cvtpd2uqq256_mask ((__v4df) __A,
  324. (__v4di) _mm256_setzero_si256(),
  325. (__mmask8) __U);
  326. }
  327. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  328. _mm_cvtps_epi64 (__m128 __A) {
  329. return (__m128i) __builtin_ia32_cvtps2qq128_mask ((__v4sf) __A,
  330. (__v2di) _mm_setzero_si128(),
  331. (__mmask8) -1);
  332. }
  333. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  334. _mm_mask_cvtps_epi64 (__m128i __W, __mmask8 __U, __m128 __A) {
  335. return (__m128i) __builtin_ia32_cvtps2qq128_mask ((__v4sf) __A,
  336. (__v2di) __W,
  337. (__mmask8) __U);
  338. }
  339. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  340. _mm_maskz_cvtps_epi64 (__mmask8 __U, __m128 __A) {
  341. return (__m128i) __builtin_ia32_cvtps2qq128_mask ((__v4sf) __A,
  342. (__v2di) _mm_setzero_si128(),
  343. (__mmask8) __U);
  344. }
  345. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  346. _mm256_cvtps_epi64 (__m128 __A) {
  347. return (__m256i) __builtin_ia32_cvtps2qq256_mask ((__v4sf) __A,
  348. (__v4di) _mm256_setzero_si256(),
  349. (__mmask8) -1);
  350. }
  351. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  352. _mm256_mask_cvtps_epi64 (__m256i __W, __mmask8 __U, __m128 __A) {
  353. return (__m256i) __builtin_ia32_cvtps2qq256_mask ((__v4sf) __A,
  354. (__v4di) __W,
  355. (__mmask8) __U);
  356. }
  357. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  358. _mm256_maskz_cvtps_epi64 (__mmask8 __U, __m128 __A) {
  359. return (__m256i) __builtin_ia32_cvtps2qq256_mask ((__v4sf) __A,
  360. (__v4di) _mm256_setzero_si256(),
  361. (__mmask8) __U);
  362. }
  363. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  364. _mm_cvtps_epu64 (__m128 __A) {
  365. return (__m128i) __builtin_ia32_cvtps2uqq128_mask ((__v4sf) __A,
  366. (__v2di) _mm_setzero_si128(),
  367. (__mmask8) -1);
  368. }
  369. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  370. _mm_mask_cvtps_epu64 (__m128i __W, __mmask8 __U, __m128 __A) {
  371. return (__m128i) __builtin_ia32_cvtps2uqq128_mask ((__v4sf) __A,
  372. (__v2di) __W,
  373. (__mmask8) __U);
  374. }
  375. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  376. _mm_maskz_cvtps_epu64 (__mmask8 __U, __m128 __A) {
  377. return (__m128i) __builtin_ia32_cvtps2uqq128_mask ((__v4sf) __A,
  378. (__v2di) _mm_setzero_si128(),
  379. (__mmask8) __U);
  380. }
  381. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  382. _mm256_cvtps_epu64 (__m128 __A) {
  383. return (__m256i) __builtin_ia32_cvtps2uqq256_mask ((__v4sf) __A,
  384. (__v4di) _mm256_setzero_si256(),
  385. (__mmask8) -1);
  386. }
  387. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  388. _mm256_mask_cvtps_epu64 (__m256i __W, __mmask8 __U, __m128 __A) {
  389. return (__m256i) __builtin_ia32_cvtps2uqq256_mask ((__v4sf) __A,
  390. (__v4di) __W,
  391. (__mmask8) __U);
  392. }
  393. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  394. _mm256_maskz_cvtps_epu64 (__mmask8 __U, __m128 __A) {
  395. return (__m256i) __builtin_ia32_cvtps2uqq256_mask ((__v4sf) __A,
  396. (__v4di) _mm256_setzero_si256(),
  397. (__mmask8) __U);
  398. }
  399. static __inline__ __m128d __DEFAULT_FN_ATTRS128
  400. _mm_cvtepi64_pd (__m128i __A) {
  401. return (__m128d)__builtin_convertvector((__v2di)__A, __v2df);
  402. }
  403. static __inline__ __m128d __DEFAULT_FN_ATTRS128
  404. _mm_mask_cvtepi64_pd (__m128d __W, __mmask8 __U, __m128i __A) {
  405. return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
  406. (__v2df)_mm_cvtepi64_pd(__A),
  407. (__v2df)__W);
  408. }
  409. static __inline__ __m128d __DEFAULT_FN_ATTRS128
  410. _mm_maskz_cvtepi64_pd (__mmask8 __U, __m128i __A) {
  411. return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
  412. (__v2df)_mm_cvtepi64_pd(__A),
  413. (__v2df)_mm_setzero_pd());
  414. }
  415. static __inline__ __m256d __DEFAULT_FN_ATTRS256
  416. _mm256_cvtepi64_pd (__m256i __A) {
  417. return (__m256d)__builtin_convertvector((__v4di)__A, __v4df);
  418. }
  419. static __inline__ __m256d __DEFAULT_FN_ATTRS256
  420. _mm256_mask_cvtepi64_pd (__m256d __W, __mmask8 __U, __m256i __A) {
  421. return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
  422. (__v4df)_mm256_cvtepi64_pd(__A),
  423. (__v4df)__W);
  424. }
  425. static __inline__ __m256d __DEFAULT_FN_ATTRS256
  426. _mm256_maskz_cvtepi64_pd (__mmask8 __U, __m256i __A) {
  427. return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
  428. (__v4df)_mm256_cvtepi64_pd(__A),
  429. (__v4df)_mm256_setzero_pd());
  430. }
  431. static __inline__ __m128 __DEFAULT_FN_ATTRS128
  432. _mm_cvtepi64_ps (__m128i __A) {
  433. return (__m128) __builtin_ia32_cvtqq2ps128_mask ((__v2di) __A,
  434. (__v4sf) _mm_setzero_ps(),
  435. (__mmask8) -1);
  436. }
  437. static __inline__ __m128 __DEFAULT_FN_ATTRS128
  438. _mm_mask_cvtepi64_ps (__m128 __W, __mmask8 __U, __m128i __A) {
  439. return (__m128) __builtin_ia32_cvtqq2ps128_mask ((__v2di) __A,
  440. (__v4sf) __W,
  441. (__mmask8) __U);
  442. }
  443. static __inline__ __m128 __DEFAULT_FN_ATTRS128
  444. _mm_maskz_cvtepi64_ps (__mmask8 __U, __m128i __A) {
  445. return (__m128) __builtin_ia32_cvtqq2ps128_mask ((__v2di) __A,
  446. (__v4sf) _mm_setzero_ps(),
  447. (__mmask8) __U);
  448. }
  449. static __inline__ __m128 __DEFAULT_FN_ATTRS256
  450. _mm256_cvtepi64_ps (__m256i __A) {
  451. return (__m128) __builtin_ia32_cvtqq2ps256_mask ((__v4di) __A,
  452. (__v4sf) _mm_setzero_ps(),
  453. (__mmask8) -1);
  454. }
  455. static __inline__ __m128 __DEFAULT_FN_ATTRS256
  456. _mm256_mask_cvtepi64_ps (__m128 __W, __mmask8 __U, __m256i __A) {
  457. return (__m128) __builtin_ia32_cvtqq2ps256_mask ((__v4di) __A,
  458. (__v4sf) __W,
  459. (__mmask8) __U);
  460. }
  461. static __inline__ __m128 __DEFAULT_FN_ATTRS256
  462. _mm256_maskz_cvtepi64_ps (__mmask8 __U, __m256i __A) {
  463. return (__m128) __builtin_ia32_cvtqq2ps256_mask ((__v4di) __A,
  464. (__v4sf) _mm_setzero_ps(),
  465. (__mmask8) __U);
  466. }
  467. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  468. _mm_cvttpd_epi64 (__m128d __A) {
  469. return (__m128i) __builtin_ia32_cvttpd2qq128_mask ((__v2df) __A,
  470. (__v2di) _mm_setzero_si128(),
  471. (__mmask8) -1);
  472. }
  473. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  474. _mm_mask_cvttpd_epi64 (__m128i __W, __mmask8 __U, __m128d __A) {
  475. return (__m128i) __builtin_ia32_cvttpd2qq128_mask ((__v2df) __A,
  476. (__v2di) __W,
  477. (__mmask8) __U);
  478. }
  479. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  480. _mm_maskz_cvttpd_epi64 (__mmask8 __U, __m128d __A) {
  481. return (__m128i) __builtin_ia32_cvttpd2qq128_mask ((__v2df) __A,
  482. (__v2di) _mm_setzero_si128(),
  483. (__mmask8) __U);
  484. }
  485. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  486. _mm256_cvttpd_epi64 (__m256d __A) {
  487. return (__m256i) __builtin_ia32_cvttpd2qq256_mask ((__v4df) __A,
  488. (__v4di) _mm256_setzero_si256(),
  489. (__mmask8) -1);
  490. }
  491. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  492. _mm256_mask_cvttpd_epi64 (__m256i __W, __mmask8 __U, __m256d __A) {
  493. return (__m256i) __builtin_ia32_cvttpd2qq256_mask ((__v4df) __A,
  494. (__v4di) __W,
  495. (__mmask8) __U);
  496. }
  497. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  498. _mm256_maskz_cvttpd_epi64 (__mmask8 __U, __m256d __A) {
  499. return (__m256i) __builtin_ia32_cvttpd2qq256_mask ((__v4df) __A,
  500. (__v4di) _mm256_setzero_si256(),
  501. (__mmask8) __U);
  502. }
  503. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  504. _mm_cvttpd_epu64 (__m128d __A) {
  505. return (__m128i) __builtin_ia32_cvttpd2uqq128_mask ((__v2df) __A,
  506. (__v2di) _mm_setzero_si128(),
  507. (__mmask8) -1);
  508. }
  509. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  510. _mm_mask_cvttpd_epu64 (__m128i __W, __mmask8 __U, __m128d __A) {
  511. return (__m128i) __builtin_ia32_cvttpd2uqq128_mask ((__v2df) __A,
  512. (__v2di) __W,
  513. (__mmask8) __U);
  514. }
  515. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  516. _mm_maskz_cvttpd_epu64 (__mmask8 __U, __m128d __A) {
  517. return (__m128i) __builtin_ia32_cvttpd2uqq128_mask ((__v2df) __A,
  518. (__v2di) _mm_setzero_si128(),
  519. (__mmask8) __U);
  520. }
  521. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  522. _mm256_cvttpd_epu64 (__m256d __A) {
  523. return (__m256i) __builtin_ia32_cvttpd2uqq256_mask ((__v4df) __A,
  524. (__v4di) _mm256_setzero_si256(),
  525. (__mmask8) -1);
  526. }
  527. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  528. _mm256_mask_cvttpd_epu64 (__m256i __W, __mmask8 __U, __m256d __A) {
  529. return (__m256i) __builtin_ia32_cvttpd2uqq256_mask ((__v4df) __A,
  530. (__v4di) __W,
  531. (__mmask8) __U);
  532. }
  533. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  534. _mm256_maskz_cvttpd_epu64 (__mmask8 __U, __m256d __A) {
  535. return (__m256i) __builtin_ia32_cvttpd2uqq256_mask ((__v4df) __A,
  536. (__v4di) _mm256_setzero_si256(),
  537. (__mmask8) __U);
  538. }
  539. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  540. _mm_cvttps_epi64 (__m128 __A) {
  541. return (__m128i) __builtin_ia32_cvttps2qq128_mask ((__v4sf) __A,
  542. (__v2di) _mm_setzero_si128(),
  543. (__mmask8) -1);
  544. }
  545. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  546. _mm_mask_cvttps_epi64 (__m128i __W, __mmask8 __U, __m128 __A) {
  547. return (__m128i) __builtin_ia32_cvttps2qq128_mask ((__v4sf) __A,
  548. (__v2di) __W,
  549. (__mmask8) __U);
  550. }
  551. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  552. _mm_maskz_cvttps_epi64 (__mmask8 __U, __m128 __A) {
  553. return (__m128i) __builtin_ia32_cvttps2qq128_mask ((__v4sf) __A,
  554. (__v2di) _mm_setzero_si128(),
  555. (__mmask8) __U);
  556. }
  557. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  558. _mm256_cvttps_epi64 (__m128 __A) {
  559. return (__m256i) __builtin_ia32_cvttps2qq256_mask ((__v4sf) __A,
  560. (__v4di) _mm256_setzero_si256(),
  561. (__mmask8) -1);
  562. }
  563. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  564. _mm256_mask_cvttps_epi64 (__m256i __W, __mmask8 __U, __m128 __A) {
  565. return (__m256i) __builtin_ia32_cvttps2qq256_mask ((__v4sf) __A,
  566. (__v4di) __W,
  567. (__mmask8) __U);
  568. }
  569. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  570. _mm256_maskz_cvttps_epi64 (__mmask8 __U, __m128 __A) {
  571. return (__m256i) __builtin_ia32_cvttps2qq256_mask ((__v4sf) __A,
  572. (__v4di) _mm256_setzero_si256(),
  573. (__mmask8) __U);
  574. }
  575. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  576. _mm_cvttps_epu64 (__m128 __A) {
  577. return (__m128i) __builtin_ia32_cvttps2uqq128_mask ((__v4sf) __A,
  578. (__v2di) _mm_setzero_si128(),
  579. (__mmask8) -1);
  580. }
  581. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  582. _mm_mask_cvttps_epu64 (__m128i __W, __mmask8 __U, __m128 __A) {
  583. return (__m128i) __builtin_ia32_cvttps2uqq128_mask ((__v4sf) __A,
  584. (__v2di) __W,
  585. (__mmask8) __U);
  586. }
  587. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  588. _mm_maskz_cvttps_epu64 (__mmask8 __U, __m128 __A) {
  589. return (__m128i) __builtin_ia32_cvttps2uqq128_mask ((__v4sf) __A,
  590. (__v2di) _mm_setzero_si128(),
  591. (__mmask8) __U);
  592. }
  593. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  594. _mm256_cvttps_epu64 (__m128 __A) {
  595. return (__m256i) __builtin_ia32_cvttps2uqq256_mask ((__v4sf) __A,
  596. (__v4di) _mm256_setzero_si256(),
  597. (__mmask8) -1);
  598. }
  599. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  600. _mm256_mask_cvttps_epu64 (__m256i __W, __mmask8 __U, __m128 __A) {
  601. return (__m256i) __builtin_ia32_cvttps2uqq256_mask ((__v4sf) __A,
  602. (__v4di) __W,
  603. (__mmask8) __U);
  604. }
  605. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  606. _mm256_maskz_cvttps_epu64 (__mmask8 __U, __m128 __A) {
  607. return (__m256i) __builtin_ia32_cvttps2uqq256_mask ((__v4sf) __A,
  608. (__v4di) _mm256_setzero_si256(),
  609. (__mmask8) __U);
  610. }
  611. static __inline__ __m128d __DEFAULT_FN_ATTRS128
  612. _mm_cvtepu64_pd (__m128i __A) {
  613. return (__m128d)__builtin_convertvector((__v2du)__A, __v2df);
  614. }
  615. static __inline__ __m128d __DEFAULT_FN_ATTRS128
  616. _mm_mask_cvtepu64_pd (__m128d __W, __mmask8 __U, __m128i __A) {
  617. return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
  618. (__v2df)_mm_cvtepu64_pd(__A),
  619. (__v2df)__W);
  620. }
  621. static __inline__ __m128d __DEFAULT_FN_ATTRS128
  622. _mm_maskz_cvtepu64_pd (__mmask8 __U, __m128i __A) {
  623. return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
  624. (__v2df)_mm_cvtepu64_pd(__A),
  625. (__v2df)_mm_setzero_pd());
  626. }
  627. static __inline__ __m256d __DEFAULT_FN_ATTRS256
  628. _mm256_cvtepu64_pd (__m256i __A) {
  629. return (__m256d)__builtin_convertvector((__v4du)__A, __v4df);
  630. }
  631. static __inline__ __m256d __DEFAULT_FN_ATTRS256
  632. _mm256_mask_cvtepu64_pd (__m256d __W, __mmask8 __U, __m256i __A) {
  633. return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
  634. (__v4df)_mm256_cvtepu64_pd(__A),
  635. (__v4df)__W);
  636. }
  637. static __inline__ __m256d __DEFAULT_FN_ATTRS256
  638. _mm256_maskz_cvtepu64_pd (__mmask8 __U, __m256i __A) {
  639. return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
  640. (__v4df)_mm256_cvtepu64_pd(__A),
  641. (__v4df)_mm256_setzero_pd());
  642. }
  643. static __inline__ __m128 __DEFAULT_FN_ATTRS128
  644. _mm_cvtepu64_ps (__m128i __A) {
  645. return (__m128) __builtin_ia32_cvtuqq2ps128_mask ((__v2di) __A,
  646. (__v4sf) _mm_setzero_ps(),
  647. (__mmask8) -1);
  648. }
  649. static __inline__ __m128 __DEFAULT_FN_ATTRS128
  650. _mm_mask_cvtepu64_ps (__m128 __W, __mmask8 __U, __m128i __A) {
  651. return (__m128) __builtin_ia32_cvtuqq2ps128_mask ((__v2di) __A,
  652. (__v4sf) __W,
  653. (__mmask8) __U);
  654. }
  655. static __inline__ __m128 __DEFAULT_FN_ATTRS128
  656. _mm_maskz_cvtepu64_ps (__mmask8 __U, __m128i __A) {
  657. return (__m128) __builtin_ia32_cvtuqq2ps128_mask ((__v2di) __A,
  658. (__v4sf) _mm_setzero_ps(),
  659. (__mmask8) __U);
  660. }
  661. static __inline__ __m128 __DEFAULT_FN_ATTRS256
  662. _mm256_cvtepu64_ps (__m256i __A) {
  663. return (__m128) __builtin_ia32_cvtuqq2ps256_mask ((__v4di) __A,
  664. (__v4sf) _mm_setzero_ps(),
  665. (__mmask8) -1);
  666. }
  667. static __inline__ __m128 __DEFAULT_FN_ATTRS256
  668. _mm256_mask_cvtepu64_ps (__m128 __W, __mmask8 __U, __m256i __A) {
  669. return (__m128) __builtin_ia32_cvtuqq2ps256_mask ((__v4di) __A,
  670. (__v4sf) __W,
  671. (__mmask8) __U);
  672. }
  673. static __inline__ __m128 __DEFAULT_FN_ATTRS256
  674. _mm256_maskz_cvtepu64_ps (__mmask8 __U, __m256i __A) {
  675. return (__m128) __builtin_ia32_cvtuqq2ps256_mask ((__v4di) __A,
  676. (__v4sf) _mm_setzero_ps(),
  677. (__mmask8) __U);
  678. }
  679. #define _mm_range_pd(A, B, C) \
  680. (__m128d)__builtin_ia32_rangepd128_mask((__v2df)(__m128d)(A), \
  681. (__v2df)(__m128d)(B), (int)(C), \
  682. (__v2df)_mm_setzero_pd(), \
  683. (__mmask8)-1)
  684. #define _mm_mask_range_pd(W, U, A, B, C) \
  685. (__m128d)__builtin_ia32_rangepd128_mask((__v2df)(__m128d)(A), \
  686. (__v2df)(__m128d)(B), (int)(C), \
  687. (__v2df)(__m128d)(W), \
  688. (__mmask8)(U))
  689. #define _mm_maskz_range_pd(U, A, B, C) \
  690. (__m128d)__builtin_ia32_rangepd128_mask((__v2df)(__m128d)(A), \
  691. (__v2df)(__m128d)(B), (int)(C), \
  692. (__v2df)_mm_setzero_pd(), \
  693. (__mmask8)(U))
  694. #define _mm256_range_pd(A, B, C) \
  695. (__m256d)__builtin_ia32_rangepd256_mask((__v4df)(__m256d)(A), \
  696. (__v4df)(__m256d)(B), (int)(C), \
  697. (__v4df)_mm256_setzero_pd(), \
  698. (__mmask8)-1)
  699. #define _mm256_mask_range_pd(W, U, A, B, C) \
  700. (__m256d)__builtin_ia32_rangepd256_mask((__v4df)(__m256d)(A), \
  701. (__v4df)(__m256d)(B), (int)(C), \
  702. (__v4df)(__m256d)(W), \
  703. (__mmask8)(U))
  704. #define _mm256_maskz_range_pd(U, A, B, C) \
  705. (__m256d)__builtin_ia32_rangepd256_mask((__v4df)(__m256d)(A), \
  706. (__v4df)(__m256d)(B), (int)(C), \
  707. (__v4df)_mm256_setzero_pd(), \
  708. (__mmask8)(U))
  709. #define _mm_range_ps(A, B, C) \
  710. (__m128)__builtin_ia32_rangeps128_mask((__v4sf)(__m128)(A), \
  711. (__v4sf)(__m128)(B), (int)(C), \
  712. (__v4sf)_mm_setzero_ps(), \
  713. (__mmask8)-1)
  714. #define _mm_mask_range_ps(W, U, A, B, C) \
  715. (__m128)__builtin_ia32_rangeps128_mask((__v4sf)(__m128)(A), \
  716. (__v4sf)(__m128)(B), (int)(C), \
  717. (__v4sf)(__m128)(W), (__mmask8)(U))
  718. #define _mm_maskz_range_ps(U, A, B, C) \
  719. (__m128)__builtin_ia32_rangeps128_mask((__v4sf)(__m128)(A), \
  720. (__v4sf)(__m128)(B), (int)(C), \
  721. (__v4sf)_mm_setzero_ps(), \
  722. (__mmask8)(U))
  723. #define _mm256_range_ps(A, B, C) \
  724. (__m256)__builtin_ia32_rangeps256_mask((__v8sf)(__m256)(A), \
  725. (__v8sf)(__m256)(B), (int)(C), \
  726. (__v8sf)_mm256_setzero_ps(), \
  727. (__mmask8)-1)
  728. #define _mm256_mask_range_ps(W, U, A, B, C) \
  729. (__m256)__builtin_ia32_rangeps256_mask((__v8sf)(__m256)(A), \
  730. (__v8sf)(__m256)(B), (int)(C), \
  731. (__v8sf)(__m256)(W), (__mmask8)(U))
  732. #define _mm256_maskz_range_ps(U, A, B, C) \
  733. (__m256)__builtin_ia32_rangeps256_mask((__v8sf)(__m256)(A), \
  734. (__v8sf)(__m256)(B), (int)(C), \
  735. (__v8sf)_mm256_setzero_ps(), \
  736. (__mmask8)(U))
  737. #define _mm_reduce_pd(A, B) \
  738. (__m128d)__builtin_ia32_reducepd128_mask((__v2df)(__m128d)(A), (int)(B), \
  739. (__v2df)_mm_setzero_pd(), \
  740. (__mmask8)-1)
  741. #define _mm_mask_reduce_pd(W, U, A, B) \
  742. (__m128d)__builtin_ia32_reducepd128_mask((__v2df)(__m128d)(A), (int)(B), \
  743. (__v2df)(__m128d)(W), \
  744. (__mmask8)(U))
  745. #define _mm_maskz_reduce_pd(U, A, B) \
  746. (__m128d)__builtin_ia32_reducepd128_mask((__v2df)(__m128d)(A), (int)(B), \
  747. (__v2df)_mm_setzero_pd(), \
  748. (__mmask8)(U))
  749. #define _mm256_reduce_pd(A, B) \
  750. (__m256d)__builtin_ia32_reducepd256_mask((__v4df)(__m256d)(A), (int)(B), \
  751. (__v4df)_mm256_setzero_pd(), \
  752. (__mmask8)-1)
  753. #define _mm256_mask_reduce_pd(W, U, A, B) \
  754. (__m256d)__builtin_ia32_reducepd256_mask((__v4df)(__m256d)(A), (int)(B), \
  755. (__v4df)(__m256d)(W), \
  756. (__mmask8)(U))
  757. #define _mm256_maskz_reduce_pd(U, A, B) \
  758. (__m256d)__builtin_ia32_reducepd256_mask((__v4df)(__m256d)(A), (int)(B), \
  759. (__v4df)_mm256_setzero_pd(), \
  760. (__mmask8)(U))
  761. #define _mm_reduce_ps(A, B) \
  762. (__m128)__builtin_ia32_reduceps128_mask((__v4sf)(__m128)(A), (int)(B), \
  763. (__v4sf)_mm_setzero_ps(), \
  764. (__mmask8)-1)
  765. #define _mm_mask_reduce_ps(W, U, A, B) \
  766. (__m128)__builtin_ia32_reduceps128_mask((__v4sf)(__m128)(A), (int)(B), \
  767. (__v4sf)(__m128)(W), \
  768. (__mmask8)(U))
  769. #define _mm_maskz_reduce_ps(U, A, B) \
  770. (__m128)__builtin_ia32_reduceps128_mask((__v4sf)(__m128)(A), (int)(B), \
  771. (__v4sf)_mm_setzero_ps(), \
  772. (__mmask8)(U))
  773. #define _mm256_reduce_ps(A, B) \
  774. (__m256)__builtin_ia32_reduceps256_mask((__v8sf)(__m256)(A), (int)(B), \
  775. (__v8sf)_mm256_setzero_ps(), \
  776. (__mmask8)-1)
  777. #define _mm256_mask_reduce_ps(W, U, A, B) \
  778. (__m256)__builtin_ia32_reduceps256_mask((__v8sf)(__m256)(A), (int)(B), \
  779. (__v8sf)(__m256)(W), \
  780. (__mmask8)(U))
  781. #define _mm256_maskz_reduce_ps(U, A, B) \
  782. (__m256)__builtin_ia32_reduceps256_mask((__v8sf)(__m256)(A), (int)(B), \
  783. (__v8sf)_mm256_setzero_ps(), \
  784. (__mmask8)(U))
  785. static __inline__ __mmask8 __DEFAULT_FN_ATTRS128
  786. _mm_movepi32_mask (__m128i __A)
  787. {
  788. return (__mmask8) __builtin_ia32_cvtd2mask128 ((__v4si) __A);
  789. }
  790. static __inline__ __mmask8 __DEFAULT_FN_ATTRS256
  791. _mm256_movepi32_mask (__m256i __A)
  792. {
  793. return (__mmask8) __builtin_ia32_cvtd2mask256 ((__v8si) __A);
  794. }
  795. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  796. _mm_movm_epi32 (__mmask8 __A)
  797. {
  798. return (__m128i) __builtin_ia32_cvtmask2d128 (__A);
  799. }
  800. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  801. _mm256_movm_epi32 (__mmask8 __A)
  802. {
  803. return (__m256i) __builtin_ia32_cvtmask2d256 (__A);
  804. }
  805. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  806. _mm_movm_epi64 (__mmask8 __A)
  807. {
  808. return (__m128i) __builtin_ia32_cvtmask2q128 (__A);
  809. }
  810. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  811. _mm256_movm_epi64 (__mmask8 __A)
  812. {
  813. return (__m256i) __builtin_ia32_cvtmask2q256 (__A);
  814. }
  815. static __inline__ __mmask8 __DEFAULT_FN_ATTRS128
  816. _mm_movepi64_mask (__m128i __A)
  817. {
  818. return (__mmask8) __builtin_ia32_cvtq2mask128 ((__v2di) __A);
  819. }
  820. static __inline__ __mmask8 __DEFAULT_FN_ATTRS256
  821. _mm256_movepi64_mask (__m256i __A)
  822. {
  823. return (__mmask8) __builtin_ia32_cvtq2mask256 ((__v4di) __A);
  824. }
  825. static __inline__ __m256 __DEFAULT_FN_ATTRS256
  826. _mm256_broadcast_f32x2 (__m128 __A)
  827. {
  828. return (__m256)__builtin_shufflevector((__v4sf)__A, (__v4sf)__A,
  829. 0, 1, 0, 1, 0, 1, 0, 1);
  830. }
  831. static __inline__ __m256 __DEFAULT_FN_ATTRS256
  832. _mm256_mask_broadcast_f32x2 (__m256 __O, __mmask8 __M, __m128 __A)
  833. {
  834. return (__m256)__builtin_ia32_selectps_256((__mmask8)__M,
  835. (__v8sf)_mm256_broadcast_f32x2(__A),
  836. (__v8sf)__O);
  837. }
  838. static __inline__ __m256 __DEFAULT_FN_ATTRS256
  839. _mm256_maskz_broadcast_f32x2 (__mmask8 __M, __m128 __A)
  840. {
  841. return (__m256)__builtin_ia32_selectps_256((__mmask8)__M,
  842. (__v8sf)_mm256_broadcast_f32x2(__A),
  843. (__v8sf)_mm256_setzero_ps());
  844. }
  845. static __inline__ __m256d __DEFAULT_FN_ATTRS256
  846. _mm256_broadcast_f64x2(__m128d __A)
  847. {
  848. return (__m256d)__builtin_shufflevector((__v2df)__A, (__v2df)__A,
  849. 0, 1, 0, 1);
  850. }
  851. static __inline__ __m256d __DEFAULT_FN_ATTRS256
  852. _mm256_mask_broadcast_f64x2(__m256d __O, __mmask8 __M, __m128d __A)
  853. {
  854. return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__M,
  855. (__v4df)_mm256_broadcast_f64x2(__A),
  856. (__v4df)__O);
  857. }
  858. static __inline__ __m256d __DEFAULT_FN_ATTRS256
  859. _mm256_maskz_broadcast_f64x2 (__mmask8 __M, __m128d __A)
  860. {
  861. return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__M,
  862. (__v4df)_mm256_broadcast_f64x2(__A),
  863. (__v4df)_mm256_setzero_pd());
  864. }
  865. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  866. _mm_broadcast_i32x2 (__m128i __A)
  867. {
  868. return (__m128i)__builtin_shufflevector((__v4si)__A, (__v4si)__A,
  869. 0, 1, 0, 1);
  870. }
  871. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  872. _mm_mask_broadcast_i32x2 (__m128i __O, __mmask8 __M, __m128i __A)
  873. {
  874. return (__m128i)__builtin_ia32_selectd_128((__mmask8)__M,
  875. (__v4si)_mm_broadcast_i32x2(__A),
  876. (__v4si)__O);
  877. }
  878. static __inline__ __m128i __DEFAULT_FN_ATTRS128
  879. _mm_maskz_broadcast_i32x2 (__mmask8 __M, __m128i __A)
  880. {
  881. return (__m128i)__builtin_ia32_selectd_128((__mmask8)__M,
  882. (__v4si)_mm_broadcast_i32x2(__A),
  883. (__v4si)_mm_setzero_si128());
  884. }
  885. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  886. _mm256_broadcast_i32x2 (__m128i __A)
  887. {
  888. return (__m256i)__builtin_shufflevector((__v4si)__A, (__v4si)__A,
  889. 0, 1, 0, 1, 0, 1, 0, 1);
  890. }
  891. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  892. _mm256_mask_broadcast_i32x2 (__m256i __O, __mmask8 __M, __m128i __A)
  893. {
  894. return (__m256i)__builtin_ia32_selectd_256((__mmask8)__M,
  895. (__v8si)_mm256_broadcast_i32x2(__A),
  896. (__v8si)__O);
  897. }
  898. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  899. _mm256_maskz_broadcast_i32x2 (__mmask8 __M, __m128i __A)
  900. {
  901. return (__m256i)__builtin_ia32_selectd_256((__mmask8)__M,
  902. (__v8si)_mm256_broadcast_i32x2(__A),
  903. (__v8si)_mm256_setzero_si256());
  904. }
  905. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  906. _mm256_broadcast_i64x2(__m128i __A)
  907. {
  908. return (__m256i)__builtin_shufflevector((__v2di)__A, (__v2di)__A,
  909. 0, 1, 0, 1);
  910. }
  911. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  912. _mm256_mask_broadcast_i64x2(__m256i __O, __mmask8 __M, __m128i __A)
  913. {
  914. return (__m256i)__builtin_ia32_selectq_256((__mmask8)__M,
  915. (__v4di)_mm256_broadcast_i64x2(__A),
  916. (__v4di)__O);
  917. }
  918. static __inline__ __m256i __DEFAULT_FN_ATTRS256
  919. _mm256_maskz_broadcast_i64x2 (__mmask8 __M, __m128i __A)
  920. {
  921. return (__m256i)__builtin_ia32_selectq_256((__mmask8)__M,
  922. (__v4di)_mm256_broadcast_i64x2(__A),
  923. (__v4di)_mm256_setzero_si256());
  924. }
  925. #define _mm256_extractf64x2_pd(A, imm) \
  926. (__m128d)__builtin_ia32_extractf64x2_256_mask((__v4df)(__m256d)(A), \
  927. (int)(imm), \
  928. (__v2df)_mm_undefined_pd(), \
  929. (__mmask8)-1)
  930. #define _mm256_mask_extractf64x2_pd(W, U, A, imm) \
  931. (__m128d)__builtin_ia32_extractf64x2_256_mask((__v4df)(__m256d)(A), \
  932. (int)(imm), \
  933. (__v2df)(__m128d)(W), \
  934. (__mmask8)(U))
  935. #define _mm256_maskz_extractf64x2_pd(U, A, imm) \
  936. (__m128d)__builtin_ia32_extractf64x2_256_mask((__v4df)(__m256d)(A), \
  937. (int)(imm), \
  938. (__v2df)_mm_setzero_pd(), \
  939. (__mmask8)(U))
  940. #define _mm256_extracti64x2_epi64(A, imm) \
  941. (__m128i)__builtin_ia32_extracti64x2_256_mask((__v4di)(__m256i)(A), \
  942. (int)(imm), \
  943. (__v2di)_mm_undefined_si128(), \
  944. (__mmask8)-1)
  945. #define _mm256_mask_extracti64x2_epi64(W, U, A, imm) \
  946. (__m128i)__builtin_ia32_extracti64x2_256_mask((__v4di)(__m256i)(A), \
  947. (int)(imm), \
  948. (__v2di)(__m128i)(W), \
  949. (__mmask8)(U))
  950. #define _mm256_maskz_extracti64x2_epi64(U, A, imm) \
  951. (__m128i)__builtin_ia32_extracti64x2_256_mask((__v4di)(__m256i)(A), \
  952. (int)(imm), \
  953. (__v2di)_mm_setzero_si128(), \
  954. (__mmask8)(U))
  955. #define _mm256_insertf64x2(A, B, imm) \
  956. (__m256d)__builtin_ia32_insertf64x2_256((__v4df)(__m256d)(A), \
  957. (__v2df)(__m128d)(B), (int)(imm))
  958. #define _mm256_mask_insertf64x2(W, U, A, B, imm) \
  959. (__m256d)__builtin_ia32_selectpd_256((__mmask8)(U), \
  960. (__v4df)_mm256_insertf64x2((A), (B), (imm)), \
  961. (__v4df)(__m256d)(W))
  962. #define _mm256_maskz_insertf64x2(U, A, B, imm) \
  963. (__m256d)__builtin_ia32_selectpd_256((__mmask8)(U), \
  964. (__v4df)_mm256_insertf64x2((A), (B), (imm)), \
  965. (__v4df)_mm256_setzero_pd())
  966. #define _mm256_inserti64x2(A, B, imm) \
  967. (__m256i)__builtin_ia32_inserti64x2_256((__v4di)(__m256i)(A), \
  968. (__v2di)(__m128i)(B), (int)(imm))
  969. #define _mm256_mask_inserti64x2(W, U, A, B, imm) \
  970. (__m256i)__builtin_ia32_selectq_256((__mmask8)(U), \
  971. (__v4di)_mm256_inserti64x2((A), (B), (imm)), \
  972. (__v4di)(__m256i)(W))
  973. #define _mm256_maskz_inserti64x2(U, A, B, imm) \
  974. (__m256i)__builtin_ia32_selectq_256((__mmask8)(U), \
  975. (__v4di)_mm256_inserti64x2((A), (B), (imm)), \
  976. (__v4di)_mm256_setzero_si256())
  977. #define _mm_mask_fpclass_pd_mask(U, A, imm) \
  978. (__mmask8)__builtin_ia32_fpclasspd128_mask((__v2df)(__m128d)(A), (int)(imm), \
  979. (__mmask8)(U))
  980. #define _mm_fpclass_pd_mask(A, imm) \
  981. (__mmask8)__builtin_ia32_fpclasspd128_mask((__v2df)(__m128d)(A), (int)(imm), \
  982. (__mmask8)-1)
  983. #define _mm256_mask_fpclass_pd_mask(U, A, imm) \
  984. (__mmask8)__builtin_ia32_fpclasspd256_mask((__v4df)(__m256d)(A), (int)(imm), \
  985. (__mmask8)(U))
  986. #define _mm256_fpclass_pd_mask(A, imm) \
  987. (__mmask8)__builtin_ia32_fpclasspd256_mask((__v4df)(__m256d)(A), (int)(imm), \
  988. (__mmask8)-1)
  989. #define _mm_mask_fpclass_ps_mask(U, A, imm) \
  990. (__mmask8)__builtin_ia32_fpclassps128_mask((__v4sf)(__m128)(A), (int)(imm), \
  991. (__mmask8)(U))
  992. #define _mm_fpclass_ps_mask(A, imm) \
  993. (__mmask8)__builtin_ia32_fpclassps128_mask((__v4sf)(__m128)(A), (int)(imm), \
  994. (__mmask8)-1)
  995. #define _mm256_mask_fpclass_ps_mask(U, A, imm) \
  996. (__mmask8)__builtin_ia32_fpclassps256_mask((__v8sf)(__m256)(A), (int)(imm), \
  997. (__mmask8)(U))
  998. #define _mm256_fpclass_ps_mask(A, imm) \
  999. (__mmask8)__builtin_ia32_fpclassps256_mask((__v8sf)(__m256)(A), (int)(imm), \
  1000. (__mmask8)-1)
  1001. #undef __DEFAULT_FN_ATTRS128
  1002. #undef __DEFAULT_FN_ATTRS256
  1003. #endif