source: Daodan/MSYS2/mingw32/lib/gcc/i686-w64-mingw32/11.2.0/include/xmmintrin.h@ 1194

Last change on this file since 1194 was 1166, checked in by rossy, 3 years ago

Daodan: Replace MinGW build env with an up-to-date MSYS2 env
File size: 43.4 KB
RevLine 
[1166]1/* Copyright (C) 2002-2021 Free Software Foundation, Inc.
2
3 This file is part of GCC.
4
5 GCC is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 3, or (at your option)
8 any later version.
9
10 GCC is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
14
15 Under Section 7 of GPL version 3, you are granted additional
16 permissions described in the GCC Runtime Library Exception, version
17 3.1, as published by the Free Software Foundation.
18
19 You should have received a copy of the GNU General Public License and
20 a copy of the GCC Runtime Library Exception along with this program;
21 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
22 <http://www.gnu.org/licenses/>. */
23
24/* Implemented from the specification included in the Intel C++ Compiler
25 User Guide and Reference, version 9.0. */
26
27#ifndef _XMMINTRIN_H_INCLUDED
28#define _XMMINTRIN_H_INCLUDED
29
30/* We need type definitions from the MMX header file. */
31#include <mmintrin.h>
32
33/* Get _mm_malloc () and _mm_free (). */
34#include <mm_malloc.h>
35
36/* Constants for use with _mm_prefetch. */
37enum _mm_hint
38{
39 /* _MM_HINT_ET is _MM_HINT_T with set 3rd bit. */
40 _MM_HINT_ET0 = 7,
41 _MM_HINT_ET1 = 6,
42 _MM_HINT_T0 = 3,
43 _MM_HINT_T1 = 2,
44 _MM_HINT_T2 = 1,
45 _MM_HINT_NTA = 0
46};
47
48/* Loads one cache line from address P to a location "closer" to the
49 processor. The selector I specifies the type of prefetch operation. */
50#ifdef __OPTIMIZE__
51extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
52_mm_prefetch (const void *__P, enum _mm_hint __I)
53{
54 __builtin_prefetch (__P, (__I & 0x4) >> 2, __I & 0x3);
55}
56#else
57#define _mm_prefetch(P, I) \
58 __builtin_prefetch ((P), ((I & 0x4) >> 2), (I & 0x3))
59#endif
60
61#ifndef __SSE__
62#pragma GCC push_options
63#pragma GCC target("sse")
64#define __DISABLE_SSE__
65#endif /* __SSE__ */
66
67/* The Intel API is flexible enough that we must allow aliasing with other
68 vector types, and their scalar components. */
69typedef float __m128 __attribute__ ((__vector_size__ (16), __may_alias__));
70
71/* Unaligned version of the same type. */
72typedef float __m128_u __attribute__ ((__vector_size__ (16), __may_alias__, __aligned__ (1)));
73
74/* Internal data types for implementing the intrinsics. */
75typedef float __v4sf __attribute__ ((__vector_size__ (16)));
76
77/* Create a selector for use with the SHUFPS instruction. */
78#define _MM_SHUFFLE(fp3,fp2,fp1,fp0) \
79 (((fp3) << 6) | ((fp2) << 4) | ((fp1) << 2) | (fp0))
80
81/* Bits in the MXCSR. */
82#define _MM_EXCEPT_MASK 0x003f
83#define _MM_EXCEPT_INVALID 0x0001
84#define _MM_EXCEPT_DENORM 0x0002
85#define _MM_EXCEPT_DIV_ZERO 0x0004
86#define _MM_EXCEPT_OVERFLOW 0x0008
87#define _MM_EXCEPT_UNDERFLOW 0x0010
88#define _MM_EXCEPT_INEXACT 0x0020
89
90#define _MM_MASK_MASK 0x1f80
91#define _MM_MASK_INVALID 0x0080
92#define _MM_MASK_DENORM 0x0100
93#define _MM_MASK_DIV_ZERO 0x0200
94#define _MM_MASK_OVERFLOW 0x0400
95#define _MM_MASK_UNDERFLOW 0x0800
96#define _MM_MASK_INEXACT 0x1000
97
98#define _MM_ROUND_MASK 0x6000
99#define _MM_ROUND_NEAREST 0x0000
100#define _MM_ROUND_DOWN 0x2000
101#define _MM_ROUND_UP 0x4000
102#define _MM_ROUND_TOWARD_ZERO 0x6000
103
104#define _MM_FLUSH_ZERO_MASK 0x8000
105#define _MM_FLUSH_ZERO_ON 0x8000
106#define _MM_FLUSH_ZERO_OFF 0x0000
107
108/* Create an undefined vector. */
109extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
110_mm_undefined_ps (void)
111{
112 __m128 __Y = __Y;
113 return __Y;
114}
115
116/* Create a vector of zeros. */
117extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
118_mm_setzero_ps (void)
119{
120 return __extension__ (__m128){ 0.0f, 0.0f, 0.0f, 0.0f };
121}
122
123/* Perform the respective operation on the lower SPFP (single-precision
124 floating-point) values of A and B; the upper three SPFP values are
125 passed through from A. */
126
127extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
128_mm_add_ss (__m128 __A, __m128 __B)
129{
130 return (__m128) __builtin_ia32_addss ((__v4sf)__A, (__v4sf)__B);
131}
132
133extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
134_mm_sub_ss (__m128 __A, __m128 __B)
135{
136 return (__m128) __builtin_ia32_subss ((__v4sf)__A, (__v4sf)__B);
137}
138
139extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
140_mm_mul_ss (__m128 __A, __m128 __B)
141{
142 return (__m128) __builtin_ia32_mulss ((__v4sf)__A, (__v4sf)__B);
143}
144
145extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
146_mm_div_ss (__m128 __A, __m128 __B)
147{
148 return (__m128) __builtin_ia32_divss ((__v4sf)__A, (__v4sf)__B);
149}
150
151extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
152_mm_sqrt_ss (__m128 __A)
153{
154 return (__m128) __builtin_ia32_sqrtss ((__v4sf)__A);
155}
156
157extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
158_mm_rcp_ss (__m128 __A)
159{
160 return (__m128) __builtin_ia32_rcpss ((__v4sf)__A);
161}
162
163extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
164_mm_rsqrt_ss (__m128 __A)
165{
166 return (__m128) __builtin_ia32_rsqrtss ((__v4sf)__A);
167}
168
169extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
170_mm_min_ss (__m128 __A, __m128 __B)
171{
172 return (__m128) __builtin_ia32_minss ((__v4sf)__A, (__v4sf)__B);
173}
174
175extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
176_mm_max_ss (__m128 __A, __m128 __B)
177{
178 return (__m128) __builtin_ia32_maxss ((__v4sf)__A, (__v4sf)__B);
179}
180
181/* Perform the respective operation on the four SPFP values in A and B. */
182
183extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
184_mm_add_ps (__m128 __A, __m128 __B)
185{
186 return (__m128) ((__v4sf)__A + (__v4sf)__B);
187}
188
189extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
190_mm_sub_ps (__m128 __A, __m128 __B)
191{
192 return (__m128) ((__v4sf)__A - (__v4sf)__B);
193}
194
195extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
196_mm_mul_ps (__m128 __A, __m128 __B)
197{
198 return (__m128) ((__v4sf)__A * (__v4sf)__B);
199}
200
201extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
202_mm_div_ps (__m128 __A, __m128 __B)
203{
204 return (__m128) ((__v4sf)__A / (__v4sf)__B);
205}
206
207extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
208_mm_sqrt_ps (__m128 __A)
209{
210 return (__m128) __builtin_ia32_sqrtps ((__v4sf)__A);
211}
212
213extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
214_mm_rcp_ps (__m128 __A)
215{
216 return (__m128) __builtin_ia32_rcpps ((__v4sf)__A);
217}
218
219extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
220_mm_rsqrt_ps (__m128 __A)
221{
222 return (__m128) __builtin_ia32_rsqrtps ((__v4sf)__A);
223}
224
225extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
226_mm_min_ps (__m128 __A, __m128 __B)
227{
228 return (__m128) __builtin_ia32_minps ((__v4sf)__A, (__v4sf)__B);
229}
230
231extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
232_mm_max_ps (__m128 __A, __m128 __B)
233{
234 return (__m128) __builtin_ia32_maxps ((__v4sf)__A, (__v4sf)__B);
235}
236
237/* Perform logical bit-wise operations on 128-bit values. */
238
239extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
240_mm_and_ps (__m128 __A, __m128 __B)
241{
242 return __builtin_ia32_andps (__A, __B);
243}
244
245extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
246_mm_andnot_ps (__m128 __A, __m128 __B)
247{
248 return __builtin_ia32_andnps (__A, __B);
249}
250
251extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
252_mm_or_ps (__m128 __A, __m128 __B)
253{
254 return __builtin_ia32_orps (__A, __B);
255}
256
257extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
258_mm_xor_ps (__m128 __A, __m128 __B)
259{
260 return __builtin_ia32_xorps (__A, __B);
261}
262
263/* Perform a comparison on the lower SPFP values of A and B. If the
264 comparison is true, place a mask of all ones in the result, otherwise a
265 mask of zeros. The upper three SPFP values are passed through from A. */
266
267extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
268_mm_cmpeq_ss (__m128 __A, __m128 __B)
269{
270 return (__m128) __builtin_ia32_cmpeqss ((__v4sf)__A, (__v4sf)__B);
271}
272
273extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
274_mm_cmplt_ss (__m128 __A, __m128 __B)
275{
276 return (__m128) __builtin_ia32_cmpltss ((__v4sf)__A, (__v4sf)__B);
277}
278
279extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
280_mm_cmple_ss (__m128 __A, __m128 __B)
281{
282 return (__m128) __builtin_ia32_cmpless ((__v4sf)__A, (__v4sf)__B);
283}
284
285extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
286_mm_cmpgt_ss (__m128 __A, __m128 __B)
287{
288 return (__m128) __builtin_ia32_movss ((__v4sf) __A,
289 (__v4sf)
290 __builtin_ia32_cmpltss ((__v4sf) __B,
291 (__v4sf)
292 __A));
293}
294
295extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
296_mm_cmpge_ss (__m128 __A, __m128 __B)
297{
298 return (__m128) __builtin_ia32_movss ((__v4sf) __A,
299 (__v4sf)
300 __builtin_ia32_cmpless ((__v4sf) __B,
301 (__v4sf)
302 __A));
303}
304
305extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
306_mm_cmpneq_ss (__m128 __A, __m128 __B)
307{
308 return (__m128) __builtin_ia32_cmpneqss ((__v4sf)__A, (__v4sf)__B);
309}
310
311extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
312_mm_cmpnlt_ss (__m128 __A, __m128 __B)
313{
314 return (__m128) __builtin_ia32_cmpnltss ((__v4sf)__A, (__v4sf)__B);
315}
316
317extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
318_mm_cmpnle_ss (__m128 __A, __m128 __B)
319{
320 return (__m128) __builtin_ia32_cmpnless ((__v4sf)__A, (__v4sf)__B);
321}
322
323extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
324_mm_cmpngt_ss (__m128 __A, __m128 __B)
325{
326 return (__m128) __builtin_ia32_movss ((__v4sf) __A,
327 (__v4sf)
328 __builtin_ia32_cmpnltss ((__v4sf) __B,
329 (__v4sf)
330 __A));
331}
332
333extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
334_mm_cmpnge_ss (__m128 __A, __m128 __B)
335{
336 return (__m128) __builtin_ia32_movss ((__v4sf) __A,
337 (__v4sf)
338 __builtin_ia32_cmpnless ((__v4sf) __B,
339 (__v4sf)
340 __A));
341}
342
343extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
344_mm_cmpord_ss (__m128 __A, __m128 __B)
345{
346 return (__m128) __builtin_ia32_cmpordss ((__v4sf)__A, (__v4sf)__B);
347}
348
349extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
350_mm_cmpunord_ss (__m128 __A, __m128 __B)
351{
352 return (__m128) __builtin_ia32_cmpunordss ((__v4sf)__A, (__v4sf)__B);
353}
354
355/* Perform a comparison on the four SPFP values of A and B. For each
356 element, if the comparison is true, place a mask of all ones in the
357 result, otherwise a mask of zeros. */
358
359extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
360_mm_cmpeq_ps (__m128 __A, __m128 __B)
361{
362 return (__m128) __builtin_ia32_cmpeqps ((__v4sf)__A, (__v4sf)__B);
363}
364
365extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
366_mm_cmplt_ps (__m128 __A, __m128 __B)
367{
368 return (__m128) __builtin_ia32_cmpltps ((__v4sf)__A, (__v4sf)__B);
369}
370
371extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
372_mm_cmple_ps (__m128 __A, __m128 __B)
373{
374 return (__m128) __builtin_ia32_cmpleps ((__v4sf)__A, (__v4sf)__B);
375}
376
377extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
378_mm_cmpgt_ps (__m128 __A, __m128 __B)
379{
380 return (__m128) __builtin_ia32_cmpgtps ((__v4sf)__A, (__v4sf)__B);
381}
382
383extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
384_mm_cmpge_ps (__m128 __A, __m128 __B)
385{
386 return (__m128) __builtin_ia32_cmpgeps ((__v4sf)__A, (__v4sf)__B);
387}
388
389extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
390_mm_cmpneq_ps (__m128 __A, __m128 __B)
391{
392 return (__m128) __builtin_ia32_cmpneqps ((__v4sf)__A, (__v4sf)__B);
393}
394
395extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
396_mm_cmpnlt_ps (__m128 __A, __m128 __B)
397{
398 return (__m128) __builtin_ia32_cmpnltps ((__v4sf)__A, (__v4sf)__B);
399}
400
401extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
402_mm_cmpnle_ps (__m128 __A, __m128 __B)
403{
404 return (__m128) __builtin_ia32_cmpnleps ((__v4sf)__A, (__v4sf)__B);
405}
406
407extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
408_mm_cmpngt_ps (__m128 __A, __m128 __B)
409{
410 return (__m128) __builtin_ia32_cmpngtps ((__v4sf)__A, (__v4sf)__B);
411}
412
413extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
414_mm_cmpnge_ps (__m128 __A, __m128 __B)
415{
416 return (__m128) __builtin_ia32_cmpngeps ((__v4sf)__A, (__v4sf)__B);
417}
418
419extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
420_mm_cmpord_ps (__m128 __A, __m128 __B)
421{
422 return (__m128) __builtin_ia32_cmpordps ((__v4sf)__A, (__v4sf)__B);
423}
424
425extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
426_mm_cmpunord_ps (__m128 __A, __m128 __B)
427{
428 return (__m128) __builtin_ia32_cmpunordps ((__v4sf)__A, (__v4sf)__B);
429}
430
431/* Compare the lower SPFP values of A and B and return 1 if true
432 and 0 if false. */
433
434extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
435_mm_comieq_ss (__m128 __A, __m128 __B)
436{
437 return __builtin_ia32_comieq ((__v4sf)__A, (__v4sf)__B);
438}
439
440extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
441_mm_comilt_ss (__m128 __A, __m128 __B)
442{
443 return __builtin_ia32_comilt ((__v4sf)__A, (__v4sf)__B);
444}
445
446extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
447_mm_comile_ss (__m128 __A, __m128 __B)
448{
449 return __builtin_ia32_comile ((__v4sf)__A, (__v4sf)__B);
450}
451
452extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
453_mm_comigt_ss (__m128 __A, __m128 __B)
454{
455 return __builtin_ia32_comigt ((__v4sf)__A, (__v4sf)__B);
456}
457
458extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
459_mm_comige_ss (__m128 __A, __m128 __B)
460{
461 return __builtin_ia32_comige ((__v4sf)__A, (__v4sf)__B);
462}
463
464extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
465_mm_comineq_ss (__m128 __A, __m128 __B)
466{
467 return __builtin_ia32_comineq ((__v4sf)__A, (__v4sf)__B);
468}
469
470extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
471_mm_ucomieq_ss (__m128 __A, __m128 __B)
472{
473 return __builtin_ia32_ucomieq ((__v4sf)__A, (__v4sf)__B);
474}
475
476extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
477_mm_ucomilt_ss (__m128 __A, __m128 __B)
478{
479 return __builtin_ia32_ucomilt ((__v4sf)__A, (__v4sf)__B);
480}
481
482extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
483_mm_ucomile_ss (__m128 __A, __m128 __B)
484{
485 return __builtin_ia32_ucomile ((__v4sf)__A, (__v4sf)__B);
486}
487
488extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
489_mm_ucomigt_ss (__m128 __A, __m128 __B)
490{
491 return __builtin_ia32_ucomigt ((__v4sf)__A, (__v4sf)__B);
492}
493
494extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
495_mm_ucomige_ss (__m128 __A, __m128 __B)
496{
497 return __builtin_ia32_ucomige ((__v4sf)__A, (__v4sf)__B);
498}
499
500extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
501_mm_ucomineq_ss (__m128 __A, __m128 __B)
502{
503 return __builtin_ia32_ucomineq ((__v4sf)__A, (__v4sf)__B);
504}
505
506/* Convert the lower SPFP value to a 32-bit integer according to the current
507 rounding mode. */
508extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
509_mm_cvtss_si32 (__m128 __A)
510{
511 return __builtin_ia32_cvtss2si ((__v4sf) __A);
512}
513
514extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
515_mm_cvt_ss2si (__m128 __A)
516{
517 return _mm_cvtss_si32 (__A);
518}
519
520#ifdef __x86_64__
521/* Convert the lower SPFP value to a 32-bit integer according to the
522 current rounding mode. */
523
524/* Intel intrinsic. */
525extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
526_mm_cvtss_si64 (__m128 __A)
527{
528 return __builtin_ia32_cvtss2si64 ((__v4sf) __A);
529}
530
531/* Microsoft intrinsic. */
532extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
533_mm_cvtss_si64x (__m128 __A)
534{
535 return __builtin_ia32_cvtss2si64 ((__v4sf) __A);
536}
537#endif
538
539/* Convert the two lower SPFP values to 32-bit integers according to the
540 current rounding mode. Return the integers in packed form. */
541extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
542_mm_cvtps_pi32 (__m128 __A)
543{
544 return (__m64) __builtin_ia32_cvtps2pi ((__v4sf) __A);
545}
546
547extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
548_mm_cvt_ps2pi (__m128 __A)
549{
550 return _mm_cvtps_pi32 (__A);
551}
552
553/* Truncate the lower SPFP value to a 32-bit integer. */
554extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
555_mm_cvttss_si32 (__m128 __A)
556{
557 return __builtin_ia32_cvttss2si ((__v4sf) __A);
558}
559
560extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
561_mm_cvtt_ss2si (__m128 __A)
562{
563 return _mm_cvttss_si32 (__A);
564}
565
566#ifdef __x86_64__
567/* Truncate the lower SPFP value to a 32-bit integer. */
568
569/* Intel intrinsic. */
570extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
571_mm_cvttss_si64 (__m128 __A)
572{
573 return __builtin_ia32_cvttss2si64 ((__v4sf) __A);
574}
575
576/* Microsoft intrinsic. */
577extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
578_mm_cvttss_si64x (__m128 __A)
579{
580 return __builtin_ia32_cvttss2si64 ((__v4sf) __A);
581}
582#endif
583
584/* Truncate the two lower SPFP values to 32-bit integers. Return the
585 integers in packed form. */
586extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
587_mm_cvttps_pi32 (__m128 __A)
588{
589 return (__m64) __builtin_ia32_cvttps2pi ((__v4sf) __A);
590}
591
592extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
593_mm_cvtt_ps2pi (__m128 __A)
594{
595 return _mm_cvttps_pi32 (__A);
596}
597
598/* Convert B to a SPFP value and insert it as element zero in A. */
599extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
600_mm_cvtsi32_ss (__m128 __A, int __B)
601{
602 return (__m128) __builtin_ia32_cvtsi2ss ((__v4sf) __A, __B);
603}
604
605extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
606_mm_cvt_si2ss (__m128 __A, int __B)
607{
608 return _mm_cvtsi32_ss (__A, __B);
609}
610
611#ifdef __x86_64__
612/* Convert B to a SPFP value and insert it as element zero in A. */
613
614/* Intel intrinsic. */
615extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
616_mm_cvtsi64_ss (__m128 __A, long long __B)
617{
618 return (__m128) __builtin_ia32_cvtsi642ss ((__v4sf) __A, __B);
619}
620
621/* Microsoft intrinsic. */
622extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
623_mm_cvtsi64x_ss (__m128 __A, long long __B)
624{
625 return (__m128) __builtin_ia32_cvtsi642ss ((__v4sf) __A, __B);
626}
627#endif
628
629/* Convert the two 32-bit values in B to SPFP form and insert them
630 as the two lower elements in A. */
631extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
632_mm_cvtpi32_ps (__m128 __A, __m64 __B)
633{
634 return (__m128) __builtin_ia32_cvtpi2ps ((__v4sf) __A, (__v2si)__B);
635}
636
637extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
638_mm_cvt_pi2ps (__m128 __A, __m64 __B)
639{
640 return _mm_cvtpi32_ps (__A, __B);
641}
642
643/* Convert the four signed 16-bit values in A to SPFP form. */
644extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
645_mm_cvtpi16_ps (__m64 __A)
646{
647 __v4hi __sign;
648 __v2si __hisi, __losi;
649 __v4sf __zero, __ra, __rb;
650
651 /* This comparison against zero gives us a mask that can be used to
652 fill in the missing sign bits in the unpack operations below, so
653 that we get signed values after unpacking. */
654 __sign = __builtin_ia32_pcmpgtw ((__v4hi)0LL, (__v4hi)__A);
655
656 /* Convert the four words to doublewords. */
657 __losi = (__v2si) __builtin_ia32_punpcklwd ((__v4hi)__A, __sign);
658 __hisi = (__v2si) __builtin_ia32_punpckhwd ((__v4hi)__A, __sign);
659
660 /* Convert the doublewords to floating point two at a time. */
661 __zero = (__v4sf) _mm_setzero_ps ();
662 __ra = __builtin_ia32_cvtpi2ps (__zero, __losi);
663 __rb = __builtin_ia32_cvtpi2ps (__ra, __hisi);
664
665 return (__m128) __builtin_ia32_movlhps (__ra, __rb);
666}
667
668/* Convert the four unsigned 16-bit values in A to SPFP form. */
669extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
670_mm_cvtpu16_ps (__m64 __A)
671{
672 __v2si __hisi, __losi;
673 __v4sf __zero, __ra, __rb;
674
675 /* Convert the four words to doublewords. */
676 __losi = (__v2si) __builtin_ia32_punpcklwd ((__v4hi)__A, (__v4hi)0LL);
677 __hisi = (__v2si) __builtin_ia32_punpckhwd ((__v4hi)__A, (__v4hi)0LL);
678
679 /* Convert the doublewords to floating point two at a time. */
680 __zero = (__v4sf) _mm_setzero_ps ();
681 __ra = __builtin_ia32_cvtpi2ps (__zero, __losi);
682 __rb = __builtin_ia32_cvtpi2ps (__ra, __hisi);
683
684 return (__m128) __builtin_ia32_movlhps (__ra, __rb);
685}
686
687/* Convert the low four signed 8-bit values in A to SPFP form. */
688extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
689_mm_cvtpi8_ps (__m64 __A)
690{
691 __v8qi __sign;
692
693 /* This comparison against zero gives us a mask that can be used to
694 fill in the missing sign bits in the unpack operations below, so
695 that we get signed values after unpacking. */
696 __sign = __builtin_ia32_pcmpgtb ((__v8qi)0LL, (__v8qi)__A);
697
698 /* Convert the four low bytes to words. */
699 __A = (__m64) __builtin_ia32_punpcklbw ((__v8qi)__A, __sign);
700
701 return _mm_cvtpi16_ps(__A);
702}
703
704/* Convert the low four unsigned 8-bit values in A to SPFP form. */
705extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
706_mm_cvtpu8_ps(__m64 __A)
707{
708 __A = (__m64) __builtin_ia32_punpcklbw ((__v8qi)__A, (__v8qi)0LL);
709 return _mm_cvtpu16_ps(__A);
710}
711
712/* Convert the four signed 32-bit values in A and B to SPFP form. */
713extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
714_mm_cvtpi32x2_ps(__m64 __A, __m64 __B)
715{
716 __v4sf __zero = (__v4sf) _mm_setzero_ps ();
717 __v4sf __sfa = __builtin_ia32_cvtpi2ps (__zero, (__v2si)__A);
718 __v4sf __sfb = __builtin_ia32_cvtpi2ps (__sfa, (__v2si)__B);
719 return (__m128) __builtin_ia32_movlhps (__sfa, __sfb);
720}
721
722/* Convert the four SPFP values in A to four signed 16-bit integers. */
723extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
724_mm_cvtps_pi16(__m128 __A)
725{
726 __v4sf __hisf = (__v4sf)__A;
727 __v4sf __losf = __builtin_ia32_movhlps (__hisf, __hisf);
728 __v2si __hisi = __builtin_ia32_cvtps2pi (__hisf);
729 __v2si __losi = __builtin_ia32_cvtps2pi (__losf);
730 return (__m64) __builtin_ia32_packssdw (__hisi, __losi);
731}
732
733/* Convert the four SPFP values in A to four signed 8-bit integers. */
734extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
735_mm_cvtps_pi8(__m128 __A)
736{
737 __v4hi __tmp = (__v4hi) _mm_cvtps_pi16 (__A);
738 return (__m64) __builtin_ia32_packsswb (__tmp, (__v4hi)0LL);
739}
740
741/* Selects four specific SPFP values from A and B based on MASK. */
742#ifdef __OPTIMIZE__
743extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
744_mm_shuffle_ps (__m128 __A, __m128 __B, int const __mask)
745{
746 return (__m128) __builtin_ia32_shufps ((__v4sf)__A, (__v4sf)__B, __mask);
747}
748#else
749#define _mm_shuffle_ps(A, B, MASK) \
750 ((__m128) __builtin_ia32_shufps ((__v4sf)(__m128)(A), \
751 (__v4sf)(__m128)(B), (int)(MASK)))
752#endif
753
754/* Selects and interleaves the upper two SPFP values from A and B. */
755extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
756_mm_unpackhi_ps (__m128 __A, __m128 __B)
757{
758 return (__m128) __builtin_ia32_unpckhps ((__v4sf)__A, (__v4sf)__B);
759}
760
761/* Selects and interleaves the lower two SPFP values from A and B. */
762extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
763_mm_unpacklo_ps (__m128 __A, __m128 __B)
764{
765 return (__m128) __builtin_ia32_unpcklps ((__v4sf)__A, (__v4sf)__B);
766}
767
768/* Sets the upper two SPFP values with 64-bits of data loaded from P;
769 the lower two values are passed through from A. */
770extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
771_mm_loadh_pi (__m128 __A, __m64 const *__P)
772{
773 return (__m128) __builtin_ia32_loadhps ((__v4sf)__A, (const __v2sf *)__P);
774}
775
776/* Stores the upper two SPFP values of A into P. */
777extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
778_mm_storeh_pi (__m64 *__P, __m128 __A)
779{
780 __builtin_ia32_storehps ((__v2sf *)__P, (__v4sf)__A);
781}
782
783/* Moves the upper two values of B into the lower two values of A. */
784extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
785_mm_movehl_ps (__m128 __A, __m128 __B)
786{
787 return (__m128) __builtin_ia32_movhlps ((__v4sf)__A, (__v4sf)__B);
788}
789
790/* Moves the lower two values of B into the upper two values of A. */
791extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
792_mm_movelh_ps (__m128 __A, __m128 __B)
793{
794 return (__m128) __builtin_ia32_movlhps ((__v4sf)__A, (__v4sf)__B);
795}
796
797/* Sets the lower two SPFP values with 64-bits of data loaded from P;
798 the upper two values are passed through from A. */
799extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
800_mm_loadl_pi (__m128 __A, __m64 const *__P)
801{
802 return (__m128) __builtin_ia32_loadlps ((__v4sf)__A, (const __v2sf *)__P);
803}
804
805/* Stores the lower two SPFP values of A into P. */
806extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
807_mm_storel_pi (__m64 *__P, __m128 __A)
808{
809 __builtin_ia32_storelps ((__v2sf *)__P, (__v4sf)__A);
810}
811
812/* Creates a 4-bit mask from the most significant bits of the SPFP values. */
813extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
814_mm_movemask_ps (__m128 __A)
815{
816 return __builtin_ia32_movmskps ((__v4sf)__A);
817}
818
819/* Return the contents of the control register. */
820extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
821_mm_getcsr (void)
822{
823 return __builtin_ia32_stmxcsr ();
824}
825
826/* Read exception bits from the control register. */
827extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
828_MM_GET_EXCEPTION_STATE (void)
829{
830 return _mm_getcsr() & _MM_EXCEPT_MASK;
831}
832
833extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
834_MM_GET_EXCEPTION_MASK (void)
835{
836 return _mm_getcsr() & _MM_MASK_MASK;
837}
838
839extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
840_MM_GET_ROUNDING_MODE (void)
841{
842 return _mm_getcsr() & _MM_ROUND_MASK;
843}
844
845extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
846_MM_GET_FLUSH_ZERO_MODE (void)
847{
848 return _mm_getcsr() & _MM_FLUSH_ZERO_MASK;
849}
850
851/* Set the control register to I. */
852extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
853_mm_setcsr (unsigned int __I)
854{
855 __builtin_ia32_ldmxcsr (__I);
856}
857
858/* Set exception bits in the control register. */
859extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
860_MM_SET_EXCEPTION_STATE(unsigned int __mask)
861{
862 _mm_setcsr((_mm_getcsr() & ~_MM_EXCEPT_MASK) | __mask);
863}
864
865extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
866_MM_SET_EXCEPTION_MASK (unsigned int __mask)
867{
868 _mm_setcsr((_mm_getcsr() & ~_MM_MASK_MASK) | __mask);
869}
870
871extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
872_MM_SET_ROUNDING_MODE (unsigned int __mode)
873{
874 _mm_setcsr((_mm_getcsr() & ~_MM_ROUND_MASK) | __mode);
875}
876
877extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
878_MM_SET_FLUSH_ZERO_MODE (unsigned int __mode)
879{
880 _mm_setcsr((_mm_getcsr() & ~_MM_FLUSH_ZERO_MASK) | __mode);
881}
882
883/* Create a vector with element 0 as F and the rest zero. */
884extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
885_mm_set_ss (float __F)
886{
887 return __extension__ (__m128)(__v4sf){ __F, 0.0f, 0.0f, 0.0f };
888}
889
890/* Create a vector with all four elements equal to F. */
891extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
892_mm_set1_ps (float __F)
893{
894 return __extension__ (__m128)(__v4sf){ __F, __F, __F, __F };
895}
896
897extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
898_mm_set_ps1 (float __F)
899{
900 return _mm_set1_ps (__F);
901}
902
903/* Create a vector with element 0 as *P and the rest zero. */
904extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
905_mm_load_ss (float const *__P)
906{
907 return _mm_set_ss (*__P);
908}
909
910/* Create a vector with all four elements equal to *P. */
911extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
912_mm_load1_ps (float const *__P)
913{
914 return _mm_set1_ps (*__P);
915}
916
917extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
918_mm_load_ps1 (float const *__P)
919{
920 return _mm_load1_ps (__P);
921}
922
923/* Load four SPFP values from P. The address must be 16-byte aligned. */
924extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
925_mm_load_ps (float const *__P)
926{
927 return *(__m128 *)__P;
928}
929
930/* Load four SPFP values from P. The address need not be 16-byte aligned. */
931extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
932_mm_loadu_ps (float const *__P)
933{
934 return *(__m128_u *)__P;
935}
936
937/* Load four SPFP values in reverse order. The address must be aligned. */
938extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
939_mm_loadr_ps (float const *__P)
940{
941 __v4sf __tmp = *(__v4sf *)__P;
942 return (__m128) __builtin_ia32_shufps (__tmp, __tmp, _MM_SHUFFLE (0,1,2,3));
943}
944
945/* Create the vector [Z Y X W]. */
946extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
947_mm_set_ps (const float __Z, const float __Y, const float __X, const float __W)
948{
949 return __extension__ (__m128)(__v4sf){ __W, __X, __Y, __Z };
950}
951
952/* Create the vector [W X Y Z]. */
953extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
954_mm_setr_ps (float __Z, float __Y, float __X, float __W)
955{
956 return __extension__ (__m128)(__v4sf){ __Z, __Y, __X, __W };
957}
958
959/* Stores the lower SPFP value. */
960extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
961_mm_store_ss (float *__P, __m128 __A)
962{
963 *__P = ((__v4sf)__A)[0];
964}
965
966extern __inline float __attribute__((__gnu_inline__, __always_inline__, __artificial__))
967_mm_cvtss_f32 (__m128 __A)
968{
969 return ((__v4sf)__A)[0];
970}
971
972/* Store four SPFP values. The address must be 16-byte aligned. */
973extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
974_mm_store_ps (float *__P, __m128 __A)
975{
976 *(__m128 *)__P = __A;
977}
978
979/* Store four SPFP values. The address need not be 16-byte aligned. */
980extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
981_mm_storeu_ps (float *__P, __m128 __A)
982{
983 *(__m128_u *)__P = __A;
984}
985
986/* Store the lower SPFP value across four words. */
987extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
988_mm_store1_ps (float *__P, __m128 __A)
989{
990 __v4sf __va = (__v4sf)__A;
991 __v4sf __tmp = __builtin_ia32_shufps (__va, __va, _MM_SHUFFLE (0,0,0,0));
992 _mm_storeu_ps (__P, __tmp);
993}
994
995extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
996_mm_store_ps1 (float *__P, __m128 __A)
997{
998 _mm_store1_ps (__P, __A);
999}
1000
1001/* Store four SPFP values in reverse order. The address must be aligned. */
1002extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1003_mm_storer_ps (float *__P, __m128 __A)
1004{
1005 __v4sf __va = (__v4sf)__A;
1006 __v4sf __tmp = __builtin_ia32_shufps (__va, __va, _MM_SHUFFLE (0,1,2,3));
1007 _mm_store_ps (__P, __tmp);
1008}
1009
1010/* Sets the low SPFP value of A from the low value of B. */
1011extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1012_mm_move_ss (__m128 __A, __m128 __B)
1013{
1014 return (__m128) __builtin_shuffle ((__v4sf)__A, (__v4sf)__B,
1015 __extension__
1016 (__attribute__((__vector_size__ (16))) int)
1017 {4,1,2,3});
1018}
1019
1020/* Extracts one of the four words of A. The selector N must be immediate. */
1021#ifdef __OPTIMIZE__
1022extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1023_mm_extract_pi16 (__m64 const __A, int const __N)
1024{
1025 return (unsigned short) __builtin_ia32_vec_ext_v4hi ((__v4hi)__A, __N);
1026}
1027
1028extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1029_m_pextrw (__m64 const __A, int const __N)
1030{
1031 return _mm_extract_pi16 (__A, __N);
1032}
1033#else
1034#define _mm_extract_pi16(A, N) \
1035 ((int) (unsigned short) __builtin_ia32_vec_ext_v4hi ((__v4hi)(__m64)(A), (int)(N)))
1036
1037#define _m_pextrw(A, N) _mm_extract_pi16(A, N)
1038#endif
1039
1040/* Inserts word D into one of four words of A. The selector N must be
1041 immediate. */
1042#ifdef __OPTIMIZE__
1043extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1044_mm_insert_pi16 (__m64 const __A, int const __D, int const __N)
1045{
1046 return (__m64) __builtin_ia32_vec_set_v4hi ((__v4hi)__A, __D, __N);
1047}
1048
1049extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1050_m_pinsrw (__m64 const __A, int const __D, int const __N)
1051{
1052 return _mm_insert_pi16 (__A, __D, __N);
1053}
1054#else
1055#define _mm_insert_pi16(A, D, N) \
1056 ((__m64) __builtin_ia32_vec_set_v4hi ((__v4hi)(__m64)(A), \
1057 (int)(D), (int)(N)))
1058
1059#define _m_pinsrw(A, D, N) _mm_insert_pi16(A, D, N)
1060#endif
1061
1062/* Compute the element-wise maximum of signed 16-bit values. */
1063extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1064_mm_max_pi16 (__m64 __A, __m64 __B)
1065{
1066 return (__m64) __builtin_ia32_pmaxsw ((__v4hi)__A, (__v4hi)__B);
1067}
1068
1069extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1070_m_pmaxsw (__m64 __A, __m64 __B)
1071{
1072 return _mm_max_pi16 (__A, __B);
1073}
1074
1075/* Compute the element-wise maximum of unsigned 8-bit values. */
1076extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1077_mm_max_pu8 (__m64 __A, __m64 __B)
1078{
1079 return (__m64) __builtin_ia32_pmaxub ((__v8qi)__A, (__v8qi)__B);
1080}
1081
1082extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1083_m_pmaxub (__m64 __A, __m64 __B)
1084{
1085 return _mm_max_pu8 (__A, __B);
1086}
1087
1088/* Compute the element-wise minimum of signed 16-bit values. */
1089extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1090_mm_min_pi16 (__m64 __A, __m64 __B)
1091{
1092 return (__m64) __builtin_ia32_pminsw ((__v4hi)__A, (__v4hi)__B);
1093}
1094
1095extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1096_m_pminsw (__m64 __A, __m64 __B)
1097{
1098 return _mm_min_pi16 (__A, __B);
1099}
1100
1101/* Compute the element-wise minimum of unsigned 8-bit values. */
1102extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1103_mm_min_pu8 (__m64 __A, __m64 __B)
1104{
1105 return (__m64) __builtin_ia32_pminub ((__v8qi)__A, (__v8qi)__B);
1106}
1107
1108extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1109_m_pminub (__m64 __A, __m64 __B)
1110{
1111 return _mm_min_pu8 (__A, __B);
1112}
1113
1114/* Create an 8-bit mask of the signs of 8-bit values. */
1115extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1116_mm_movemask_pi8 (__m64 __A)
1117{
1118 return __builtin_ia32_pmovmskb ((__v8qi)__A);
1119}
1120
1121extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1122_m_pmovmskb (__m64 __A)
1123{
1124 return _mm_movemask_pi8 (__A);
1125}
1126
1127/* Multiply four unsigned 16-bit values in A by four unsigned 16-bit values
1128 in B and produce the high 16 bits of the 32-bit results. */
1129extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1130_mm_mulhi_pu16 (__m64 __A, __m64 __B)
1131{
1132 return (__m64) __builtin_ia32_pmulhuw ((__v4hi)__A, (__v4hi)__B);
1133}
1134
1135extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1136_m_pmulhuw (__m64 __A, __m64 __B)
1137{
1138 return _mm_mulhi_pu16 (__A, __B);
1139}
1140
1141/* Return a combination of the four 16-bit values in A. The selector
1142 must be an immediate. */
1143#ifdef __OPTIMIZE__
1144extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1145_mm_shuffle_pi16 (__m64 __A, int const __N)
1146{
1147 return (__m64) __builtin_ia32_pshufw ((__v4hi)__A, __N);
1148}
1149
1150extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1151_m_pshufw (__m64 __A, int const __N)
1152{
1153 return _mm_shuffle_pi16 (__A, __N);
1154}
1155#else
1156#define _mm_shuffle_pi16(A, N) \
1157 ((__m64) __builtin_ia32_pshufw ((__v4hi)(__m64)(A), (int)(N)))
1158
1159#define _m_pshufw(A, N) _mm_shuffle_pi16 (A, N)
1160#endif
1161
1162/* Conditionally store byte elements of A into P. The high bit of each
1163 byte in the selector N determines whether the corresponding byte from
1164 A is stored. */
1165extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1166_mm_maskmove_si64 (__m64 __A, __m64 __N, char *__P)
1167{
1168#ifdef __MMX_WITH_SSE__
1169 /* Emulate MMX maskmovq with SSE2 maskmovdqu and handle unmapped bits
1170 64:127 at address __P. */
1171 typedef long long __v2di __attribute__ ((__vector_size__ (16)));
1172 typedef char __v16qi __attribute__ ((__vector_size__ (16)));
1173 /* Zero-extend __A and __N to 128 bits. */
1174 __v2di __A128 = __extension__ (__v2di) { ((__v1di) __A)[0], 0 };
1175 __v2di __N128 = __extension__ (__v2di) { ((__v1di) __N)[0], 0 };
1176
1177 /* Check the alignment of __P. */
1178 __SIZE_TYPE__ offset = ((__SIZE_TYPE__) __P) & 0xf;
1179 if (offset)
1180 {
1181 /* If the misalignment of __P > 8, subtract __P by 8 bytes.
1182 Otherwise, subtract __P by the misalignment. */
1183 if (offset > 8)
1184 offset = 8;
1185 __P = (char *) (((__SIZE_TYPE__) __P) - offset);
1186
1187 /* Shift __A128 and __N128 to the left by the adjustment. */
1188 switch (offset)
1189 {
1190 case 1:
1191 __A128 = __builtin_ia32_pslldqi128 (__A128, 8);
1192 __N128 = __builtin_ia32_pslldqi128 (__N128, 8);
1193 break;
1194 case 2:
1195 __A128 = __builtin_ia32_pslldqi128 (__A128, 2 * 8);
1196 __N128 = __builtin_ia32_pslldqi128 (__N128, 2 * 8);
1197 break;
1198 case 3:
1199 __A128 = __builtin_ia32_pslldqi128 (__A128, 3 * 8);
1200 __N128 = __builtin_ia32_pslldqi128 (__N128, 3 * 8);
1201 break;
1202 case 4:
1203 __A128 = __builtin_ia32_pslldqi128 (__A128, 4 * 8);
1204 __N128 = __builtin_ia32_pslldqi128 (__N128, 4 * 8);
1205 break;
1206 case 5:
1207 __A128 = __builtin_ia32_pslldqi128 (__A128, 5 * 8);
1208 __N128 = __builtin_ia32_pslldqi128 (__N128, 5 * 8);
1209 break;
1210 case 6:
1211 __A128 = __builtin_ia32_pslldqi128 (__A128, 6 * 8);
1212 __N128 = __builtin_ia32_pslldqi128 (__N128, 6 * 8);
1213 break;
1214 case 7:
1215 __A128 = __builtin_ia32_pslldqi128 (__A128, 7 * 8);
1216 __N128 = __builtin_ia32_pslldqi128 (__N128, 7 * 8);
1217 break;
1218 case 8:
1219 __A128 = __builtin_ia32_pslldqi128 (__A128, 8 * 8);
1220 __N128 = __builtin_ia32_pslldqi128 (__N128, 8 * 8);
1221 break;
1222 default:
1223 break;
1224 }
1225 }
1226 __builtin_ia32_maskmovdqu ((__v16qi)__A128, (__v16qi)__N128, __P);
1227#else
1228 __builtin_ia32_maskmovq ((__v8qi)__A, (__v8qi)__N, __P);
1229#endif
1230}
1231
1232extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1233_m_maskmovq (__m64 __A, __m64 __N, char *__P)
1234{
1235 _mm_maskmove_si64 (__A, __N, __P);
1236}
1237
1238/* Compute the rounded averages of the unsigned 8-bit values in A and B. */
1239extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1240_mm_avg_pu8 (__m64 __A, __m64 __B)
1241{
1242 return (__m64) __builtin_ia32_pavgb ((__v8qi)__A, (__v8qi)__B);
1243}
1244
1245extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1246_m_pavgb (__m64 __A, __m64 __B)
1247{
1248 return _mm_avg_pu8 (__A, __B);
1249}
1250
1251/* Compute the rounded averages of the unsigned 16-bit values in A and B. */
1252extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1253_mm_avg_pu16 (__m64 __A, __m64 __B)
1254{
1255 return (__m64) __builtin_ia32_pavgw ((__v4hi)__A, (__v4hi)__B);
1256}
1257
1258extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1259_m_pavgw (__m64 __A, __m64 __B)
1260{
1261 return _mm_avg_pu16 (__A, __B);
1262}
1263
1264/* Compute the sum of the absolute differences of the unsigned 8-bit
1265 values in A and B. Return the value in the lower 16-bit word; the
1266 upper words are cleared. */
1267extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1268_mm_sad_pu8 (__m64 __A, __m64 __B)
1269{
1270 return (__m64) __builtin_ia32_psadbw ((__v8qi)__A, (__v8qi)__B);
1271}
1272
1273extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1274_m_psadbw (__m64 __A, __m64 __B)
1275{
1276 return _mm_sad_pu8 (__A, __B);
1277}
1278
1279/* Stores the data in A to the address P without polluting the caches. */
1280extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1281_mm_stream_pi (__m64 *__P, __m64 __A)
1282{
1283 __builtin_ia32_movntq ((unsigned long long *)__P, (unsigned long long)__A);
1284}
1285
1286/* Likewise. The address must be 16-byte aligned. */
1287extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1288_mm_stream_ps (float *__P, __m128 __A)
1289{
1290 __builtin_ia32_movntps (__P, (__v4sf)__A);
1291}
1292
1293/* Guarantees that every preceding store is globally visible before
1294 any subsequent store. */
1295extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1296_mm_sfence (void)
1297{
1298 __builtin_ia32_sfence ();
1299}
1300
1301/* Transpose the 4x4 matrix composed of row[0-3]. */
1302#define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) \
1303do { \
1304 __v4sf __r0 = (row0), __r1 = (row1), __r2 = (row2), __r3 = (row3); \
1305 __v4sf __t0 = __builtin_ia32_unpcklps (__r0, __r1); \
1306 __v4sf __t1 = __builtin_ia32_unpcklps (__r2, __r3); \
1307 __v4sf __t2 = __builtin_ia32_unpckhps (__r0, __r1); \
1308 __v4sf __t3 = __builtin_ia32_unpckhps (__r2, __r3); \
1309 (row0) = __builtin_ia32_movlhps (__t0, __t1); \
1310 (row1) = __builtin_ia32_movhlps (__t1, __t0); \
1311 (row2) = __builtin_ia32_movlhps (__t2, __t3); \
1312 (row3) = __builtin_ia32_movhlps (__t3, __t2); \
1313} while (0)
1314
1315/* For backward source compatibility. */
1316# include <emmintrin.h>
1317
1318#ifdef __DISABLE_SSE__
1319#undef __DISABLE_SSE__
1320#pragma GCC pop_options
1321#endif /* __DISABLE_SSE__ */
1322
1323/* The execution of the next instruction is delayed by an implementation
1324 specific amount of time. The instruction does not modify the
1325 architectural state. This is after the pop_options pragma because
1326 it does not require SSE support in the processor--the encoding is a
1327 nop on processors that do not support it. */
1328extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1329_mm_pause (void)
1330{
1331 __builtin_ia32_pause ();
1332}
1333
1334#endif /* _XMMINTRIN_H_INCLUDED */
Note: See TracBrowser for help on using the repository browser.