![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to rshift.asm CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [local] / OpenXM_contrib / gmp / mpn / x86 / k7 / mmx|
Return to rshift.asm CVS log | Up to [local] / OpenXM_contrib / gmp / mpn / x86 / k7 / mmx |
1.1 maekawa 1: dnl AMD K7 mpn_rshift -- mpn right shift.
2: dnl
3: dnl K7: 1.21 cycles/limb (at 16 limbs/loop).
4:
5:
6: dnl Copyright (C) 1999, 2000 Free Software Foundation, Inc.
7: dnl
8: dnl This file is part of the GNU MP Library.
9: dnl
10: dnl The GNU MP Library is free software; you can redistribute it and/or
11: dnl modify it under the terms of the GNU Lesser General Public License as
12: dnl published by the Free Software Foundation; either version 2.1 of the
13: dnl License, or (at your option) any later version.
14: dnl
15: dnl The GNU MP Library is distributed in the hope that it will be useful,
16: dnl but WITHOUT ANY WARRANTY; without even the implied warranty of
17: dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18: dnl Lesser General Public License for more details.
19: dnl
20: dnl You should have received a copy of the GNU Lesser General Public
21: dnl License along with the GNU MP Library; see the file COPYING.LIB. If
22: dnl not, write to the Free Software Foundation, Inc., 59 Temple Place -
23: dnl Suite 330, Boston, MA 02111-1307, USA.
24:
25:
26: include(`../config.m4')
27:
28:
29: dnl K7: UNROLL_COUNT cycles/limb
30: dnl 4 1.51
31: dnl 8 1.26
32: dnl 16 1.21
33: dnl 32 1.2
34: dnl Maximum possible with the current code is 64.
35:
36: deflit(UNROLL_COUNT, 16)
37:
38:
39: C mp_limb_t mpn_rshift (mp_ptr dst, mp_srcptr src, mp_size_t size,
40: C unsigned shift);
41: C
42: C Shift src,size right by shift many bits and store the result in dst,size.
43: C Zeros are shifted in at the left. The bits shifted out at the right are
44: C the return value.
45: C
46: C This code uses 64-bit MMX operations, which makes it possible to handle
47: C two limbs at a time, for a theoretical 1.0 cycles/limb. Plain integer
48: C code, on the other hand, suffers from shrd being a vector path decode and
49: C running at 3 cycles back-to-back.
50: C
51: C Full speed depends on source and destination being aligned, and some hairy
52: C setups and finish-ups are done to arrange this for the loop.
53:
54: ifdef(`PIC',`
55: deflit(UNROLL_THRESHOLD, 10)
56: ',`
57: deflit(UNROLL_THRESHOLD, 10)
58: ')
59:
60: defframe(PARAM_SHIFT,16)
61: defframe(PARAM_SIZE, 12)
62: defframe(PARAM_SRC, 8)
63: defframe(PARAM_DST, 4)
64:
65: defframe(SAVE_EDI, -4)
66: defframe(SAVE_ESI, -8)
67: defframe(SAVE_EBX, -12)
68: deflit(SAVE_SIZE, 12)
69:
70: .text
71: ALIGN(32)
72:
73: PROLOGUE(mpn_rshift)
74: deflit(`FRAME',0)
75:
76: movl PARAM_SIZE, %eax
77: movl PARAM_SRC, %edx
78: subl $SAVE_SIZE, %esp
79: deflit(`FRAME',SAVE_SIZE)
80:
81: movl PARAM_SHIFT, %ecx
82: movl %edi, SAVE_EDI
83:
84: movl PARAM_DST, %edi
85: decl %eax
86: jnz L(more_than_one_limb)
87:
88: movl (%edx), %edx C src limb
89:
90: shrdl( %cl, %edx, %eax) C eax was decremented to zero
91:
92: shrl %cl, %edx
93:
94: movl %edx, (%edi) C dst limb
95: movl SAVE_EDI, %edi
96: addl $SAVE_SIZE, %esp
97:
98: ret
99:
100:
101: C -----------------------------------------------------------------------------
102: L(more_than_one_limb):
103: C eax size-1
104: C ebx
105: C ecx shift
106: C edx src
107: C esi
108: C edi dst
109: C ebp
110:
111: movd PARAM_SHIFT, %mm6 C rshift
112: movd (%edx), %mm5 C src low limb
113: cmp $UNROLL_THRESHOLD-1, %eax
114:
115: jae L(unroll)
116: leal (%edx,%eax,4), %edx C &src[size-1]
117: leal -4(%edi,%eax,4), %edi C &dst[size-2]
118:
119: movd (%edx), %mm4 C src high limb
120: negl %eax
121:
122:
123: L(simple_top):
124: C eax loop counter, limbs, negative
125: C ebx
126: C ecx shift
127: C edx carry
128: C edx &src[size-1]
129: C edi &dst[size-2]
130: C ebp
131: C
132: C mm0 scratch
133: C mm4 src high limb
134: C mm5 src low limb
135: C mm6 shift
136:
137: movq (%edx,%eax,4), %mm0
138: incl %eax
139:
140: psrlq %mm6, %mm0
141:
142: movd %mm0, (%edi,%eax,4)
143: jnz L(simple_top)
144:
145:
146: psllq $32, %mm5
147: psrlq %mm6, %mm4
148:
149: psrlq %mm6, %mm5
150: movd %mm4, 4(%edi) C dst high limb
151:
152: movd %mm5, %eax C return value
153:
154: movl SAVE_EDI, %edi
155: addl $SAVE_SIZE, %esp
156: emms
157:
158: ret
159:
160:
161: C -----------------------------------------------------------------------------
162: ALIGN(16)
163: L(unroll):
164: C eax size-1
165: C ebx
166: C ecx shift
167: C edx src
168: C esi
169: C edi dst
170: C ebp
171: C
172: C mm5 src low limb
173: C mm6 rshift
174:
175: testb $4, %dl
176: movl %esi, SAVE_ESI
177: movl %ebx, SAVE_EBX
178:
179: psllq $32, %mm5
180: jz L(start_src_aligned)
181:
182:
183: C src isn't aligned, process low limb separately (marked xxx) and
184: C step src and dst by one limb, making src aligned.
185: C
186: C source edx
187: C --+-------+-------+-------+
188: C | xxx |
189: C --+-------+-------+-------+
190: C 4mod8 0mod8 4mod8
191: C
192: C dest edi
193: C --+-------+-------+
194: C | | xxx |
195: C --+-------+-------+
196:
197: movq (%edx), %mm0 C src low two limbs
198: addl $4, %edx
199: movl %eax, PARAM_SIZE C size-1
200:
201: addl $4, %edi
202: decl %eax C size-2 is new size-1
203:
204: psrlq %mm6, %mm0
205: movl %edi, PARAM_DST C new dst
206:
207: movd %mm0, -4(%edi)
208: L(start_src_aligned):
209:
210:
211: movq (%edx), %mm1 C src low two limbs
212: decl %eax C size-2, two last limbs handled at end
213: testl $4, %edi
214:
215: psrlq %mm6, %mm5
216: jz L(start_dst_aligned)
217:
218:
219: C dst isn't aligned, add 4 to make it so, and pretend the shift is
220: C 32 bits extra. Low limb of dst (marked xxx) handled here separately.
221: C
222: C source edx
223: C --+-------+-------+
224: C | mm1 |
225: C --+-------+-------+
226: C 4mod8 0mod8
227: C
228: C dest edi
229: C --+-------+-------+-------+
230: C | xxx |
231: C --+-------+-------+-------+
232: C 4mod8 0mod8 4mod8
233:
234: movq %mm1, %mm0
235: psrlq %mm6, %mm1
236: addl $32, %ecx C shift+32
237:
238: movd %mm1, (%edi)
239: movq %mm0, %mm1
240: addl $4, %edi C new dst
241:
242: movd %ecx, %mm6
243: L(start_dst_aligned):
244:
245:
246: movq %mm1, %mm2 C copy of src low two limbs
247: negl %ecx
248: andl $-2, %eax C round size down to even
249:
250: movl %eax, %ebx
251: negl %eax
252: addl $64, %ecx
253:
254: andl $UNROLL_MASK, %eax
255: decl %ebx
256:
257: shll %eax
258:
259: movd %ecx, %mm7 C lshift = 64-rshift
260:
261: ifdef(`PIC',`
262: call L(pic_calc)
263: L(here):
264: ',`
265: leal L(entry) (%eax,%eax,4), %esi
266: negl %eax
267: ')
268: shrl $UNROLL_LOG2, %ebx C loop counter
269:
270: leal ifelse(UNROLL_BYTES,256,128+) 8(%edx,%eax,2), %edx
271: leal ifelse(UNROLL_BYTES,256,128) (%edi,%eax,2), %edi
272: movl PARAM_SIZE, %eax C for use at end
273:
274: jmp *%esi
275:
276:
277: ifdef(`PIC',`
278: L(pic_calc):
279: C See README.family about old gas bugs
280: leal (%eax,%eax,4), %esi
281: addl $L(entry)-L(here), %esi
282: addl (%esp), %esi
283: negl %eax
284:
285: ret
286: ')
287:
288:
289: C -----------------------------------------------------------------------------
290: ALIGN(64)
291: L(top):
292: C eax size, for use at end
293: C ebx loop counter
294: C ecx lshift
295: C edx src
296: C esi was computed jump
297: C edi dst
298: C ebp
299: C
300: C mm0 scratch
301: C mm1 \ carry (alternating)
302: C mm2 /
303: C mm6 rshift
304: C mm7 lshift
305: C
306: C 10 code bytes/limb
307: C
308: C The two chunks differ in whether mm1 or mm2 hold the carry.
309: C The computed jump puts the initial carry in both mm1 and mm2.
310:
311: L(entry):
312: deflit(CHUNK_COUNT, 4)
313: forloop(i, 0, UNROLL_COUNT/CHUNK_COUNT-1, `
314: deflit(`disp0', eval(i*CHUNK_COUNT*4 ifelse(UNROLL_BYTES,256,-128)))
315: deflit(`disp1', eval(disp0 + 8))
316:
317: movq disp0(%edx), %mm0
318: psrlq %mm6, %mm2
319:
320: movq %mm0, %mm1
321: psllq %mm7, %mm0
322:
323: por %mm2, %mm0
324: movq %mm0, disp0(%edi)
325:
326:
327: movq disp1(%edx), %mm0
328: psrlq %mm6, %mm1
329:
330: movq %mm0, %mm2
331: psllq %mm7, %mm0
332:
333: por %mm1, %mm0
334: movq %mm0, disp1(%edi)
335: ')
336:
337: addl $UNROLL_BYTES, %edx
338: addl $UNROLL_BYTES, %edi
339: decl %ebx
340:
341: jns L(top)
342:
343:
344: deflit(`disp0', ifelse(UNROLL_BYTES,256,-128))
345: deflit(`disp1', eval(disp0-0 + 8))
346:
347: testb $1, %al
348: psrlq %mm6, %mm2 C wanted rshifted in all cases below
349: movl SAVE_ESI, %esi
350:
351: movd %mm5, %eax C return value
352:
353: movl SAVE_EBX, %ebx
354: jz L(end_even)
355:
356:
357: C Size odd, destination was aligned.
358: C
359: C source
360: C edx
361: C +-------+---------------+--
362: C | | mm2 |
363: C +-------+---------------+--
364: C
365: C dest edi
366: C +-------+---------------+---------------+--
367: C | | | written |
368: C +-------+---------------+---------------+--
369: C
370: C mm6 = shift
371: C mm7 = ecx = 64-shift
372:
373:
374: C Size odd, destination was unaligned.
375: C
376: C source
377: C edx
378: C +-------+---------------+--
379: C | | mm2 |
380: C +-------+---------------+--
381: C
382: C dest edi
383: C +---------------+---------------+--
384: C | | written |
385: C +---------------+---------------+--
386: C
387: C mm6 = shift+32
388: C mm7 = ecx = 64-(shift+32)
389:
390:
391: C In both cases there's one extra limb of src to fetch and combine
392: C with mm2 to make a qword to store, and in the aligned case there's
393: C a further extra limb of dst to be formed.
394:
395:
396: movd disp0(%edx), %mm0
397: movq %mm0, %mm1
398:
399: psllq %mm7, %mm0
400: testb $32, %cl
401:
402: por %mm2, %mm0
403: psrlq %mm6, %mm1
404:
405: movq %mm0, disp0(%edi)
406: jz L(finish_odd_unaligned)
407:
408: movd %mm1, disp1(%edi)
409: L(finish_odd_unaligned):
410:
411: movl SAVE_EDI, %edi
412: addl $SAVE_SIZE, %esp
413: emms
414:
415: ret
416:
417:
418: L(end_even):
419:
420: C Size even, destination was aligned.
421: C
422: C source
423: C +---------------+--
424: C | mm2 |
425: C +---------------+--
426: C
427: C dest edi
428: C +---------------+---------------+--
429: C | | mm3 |
430: C +---------------+---------------+--
431: C
432: C mm6 = shift
433: C mm7 = ecx = 64-shift
434:
435:
436: C Size even, destination was unaligned.
437: C
438: C source
439: C +---------------+--
440: C | mm2 |
441: C +---------------+--
442: C
443: C dest edi
444: C +-------+---------------+--
445: C | | mm3 |
446: C +-------+---------------+--
447: C
448: C mm6 = shift+32
449: C mm7 = 64-(shift+32)
450:
451:
452: C The movd for the unaligned case is the same data as the movq for
453: C the aligned case, it's just a choice between whether one or two
454: C limbs should be written.
455:
456:
457: testb $32, %cl
458: movd %mm2, disp0(%edi)
459:
460: jz L(end_even_unaligned)
461:
462: movq %mm2, disp0(%edi)
463: L(end_even_unaligned):
464:
465: movl SAVE_EDI, %edi
466: addl $SAVE_SIZE, %esp
467: emms
468:
469: ret
470:
471: EPILOGUE()