dnl  Intel Pentium MMX mpn_mul_1 -- mpn by limb multiplication.

dnl  Copyright 2000-2002 Free Software Foundation, Inc.

dnl  This file is part of the GNU MP Library.
dnl
dnl  The GNU MP Library is free software; you can redistribute it and/or modify
dnl  it under the terms of either:
dnl
dnl    * the GNU Lesser General Public License as published by the Free
dnl      Software Foundation; either version 3 of the License, or (at your
dnl      option) any later version.
dnl
dnl  or
dnl
dnl    * the GNU General Public License as published by the Free Software
dnl      Foundation; either version 2 of the License, or (at your option) any
dnl      later version.
dnl
dnl  or both in parallel, as here.
dnl
dnl  The GNU MP Library is distributed in the hope that it will be useful, but
dnl  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
dnl  or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
dnl  for more details.
dnl
dnl  You should have received copies of the GNU General Public License and the
dnl  GNU Lesser General Public License along with the GNU MP Library.  If not,
dnl  see https://www.gnu.org/licenses/.

include(`../config.m4')


C    cycles/limb
C P5:   12.0   for 32-bit multiplier
C        7.0   for 16-bit multiplier


C mp_limb_t mpn_mul_1 (mp_ptr dst, mp_srcptr src, mp_size_t size,
C                      mp_limb_t multiplier);
C
C When the multiplier is 16 bits some special case MMX code is used.  Small
C multipliers might arise reasonably often from mpz_mul_ui etc.  If the size
C is odd there's roughly a 5 cycle penalty, so times for say size==7 and
C size==8 end up being quite close.  If src isn't aligned to an 8 byte
C boundary then one limb is processed separately with roughly a 5 cycle
C penalty, so in that case it's say size==8 and size==9 which are close.
C
C Alternatives:
C
C MMX is not believed to be of any use for 32-bit multipliers, since for
C instance the current method would just have to be more or less duplicated
C for the high and low halves of the multiplier, and would probably
C therefore run at about 14 cycles, which is slower than the plain integer
C at 12.
C
C Adding the high and low MMX products using integer code seems best.  An
C attempt at using paddd and carry bit propagation with pcmpgtd didn't give
C any joy.  Perhaps something could be done keeping the values signed and
C thereby avoiding adjustments to make pcmpgtd into an unsigned compare, or
C perhaps not.
C
C Future:
C
C An mpn_mul_1c entrypoint would need a double carry out of the low result
C limb in the 16-bit code, unless it could be assumed the carry fits in 16
C bits, possibly as carry<multiplier, this being true of a big calculation
C done piece by piece.  But let's worry about that if/when mul_1c is
C actually used.

defframe(PARAM_MULTIPLIER,16)
defframe(PARAM_SIZE,      12)
defframe(PARAM_SRC,       8)
defframe(PARAM_DST,       4)

	TEXT

	ALIGN(8)
PROLOGUE(mpn_mul_1)
deflit(`FRAME',0)

	movl	PARAM_SIZE, %ecx
	movl	PARAM_SRC, %edx

	cmpl	$1, %ecx
	jne	L(two_or_more)

	C one limb only

	movl	PARAM_MULTIPLIER, %eax
	movl	PARAM_DST, %ecx

	mull	(%edx)

	movl	%eax, (%ecx)
	movl	%edx, %eax

	ret


L(two_or_more):
	C eax	size
	C ebx
	C ecx	carry
	C edx
	C esi	src
	C edi
	C ebp

	pushl	%esi		FRAME_pushl()
	pushl	%edi		FRAME_pushl()

	movl	%edx, %esi		C src
	movl	PARAM_DST, %edi

	movl	PARAM_MULTIPLIER, %eax
	pushl	%ebx		FRAME_pushl()

	leal	(%esi,%ecx,4), %esi	C src end
	leal	(%edi,%ecx,4), %edi	C dst end

	negl	%ecx			C -size

	pushl	%ebp		FRAME_pushl()
	cmpl	$65536, %eax

	jb	L(small)


L(big):
	xorl	%ebx, %ebx		C carry limb
	sarl	%ecx			C -size/2

	jnc	L(top)			C with carry flag clear


	C size was odd, process one limb separately

	mull	4(%esi,%ecx,8)		C m * src[0]

	movl	%eax, 4(%edi,%ecx,8)
	incl	%ecx

	orl	%edx, %ebx		C carry limb, and clear carry flag


L(top):
	C eax
	C ebx	carry
	C ecx	counter, negative
	C edx
	C esi	src end
	C edi	dst end
	C ebp	(scratch carry)

	adcl	$0, %ebx
	movl	(%esi,%ecx,8), %eax

	mull	PARAM_MULTIPLIER

	movl	%edx, %ebp
	addl	%eax, %ebx

	adcl	$0, %ebp
	movl	4(%esi,%ecx,8), %eax

	mull	PARAM_MULTIPLIER

	movl	%ebx, (%edi,%ecx,8)
	addl	%ebp, %eax

	movl	%eax, 4(%edi,%ecx,8)
	incl	%ecx

	movl	%edx, %ebx
	jnz	L(top)


	adcl	$0, %ebx
	popl	%ebp

	movl	%ebx, %eax
	popl	%ebx

	popl	%edi
	popl	%esi

	ret


L(small):
	C Special case for 16-bit multiplier.
	C
	C eax	multiplier
	C ebx
	C ecx	-size
	C edx	src
	C esi	src end
	C edi	dst end
	C ebp	multiplier

	C size<3 not supported here.  At size==3 we're already a couple of
	C cycles faster, so there's no threshold as such, just use the MMX
	C as soon as possible.

	cmpl	$-3, %ecx
	ja	L(big)

	movd	%eax, %mm7		C m
	pxor	%mm6, %mm6		C initial carry word

	punpcklwd %mm7, %mm7		C m replicated 2 times
	addl	$2, %ecx		C -size+2

	punpckldq %mm7, %mm7		C m replicated 4 times
	andl	$4, %edx		C test alignment, clear carry flag

	movq	%mm7, %mm0		C m
	jz	L(small_entry)


	C Source is unaligned, process one limb separately.
	C
	C Plain integer code is used here, since it's smaller and is about
	C the same 13 cycles as an mmx block would be.
	C
	C An "addl $1,%ecx" doesn't clear the carry flag when size==3, hence
	C the use of separate incl and orl.

	mull	-8(%esi,%ecx,4)		C m * src[0]

	movl	%eax, -8(%edi,%ecx,4)	C dst[0]
	incl	%ecx			C one limb processed

	movd	%edx, %mm6		C initial carry

	orl	%eax, %eax		C clear carry flag
	jmp	L(small_entry)


C The scheduling here is quite tricky, since so many instructions have
C pairing restrictions.  In particular the js won't pair with a movd, and
C can't be paired with an adc since it wants flags from the inc, so
C instructions are rotated to the top of the loop to find somewhere useful
C for it.
C
C Trouble has been taken to avoid overlapping successive loop iterations,
C since that would greatly increase the size of the startup and finishup
C code.  Actually there's probably not much advantage to be had from
C overlapping anyway, since the difficulties are mostly with pairing, not
C with latencies as such.
C
C In the comments x represents the src data and m the multiplier (16
C bits, but replicated 4 times).
C
C The m signs calculated in %mm3 are a loop invariant and could be held in
C say %mm5, but that would save only one instruction and hence be no faster.

L(small_top):
	C eax	l.low, then l.high
	C ebx	(h.low)
	C ecx	counter, -size+2 to 0 or 1
	C edx	(h.high)
	C esi	&src[size]
	C edi	&dst[size]
	C ebp
	C
	C %mm0	(high products)
	C %mm1	(low products)
	C %mm2	(adjust for m using x signs)
	C %mm3	(adjust for x using m signs)
	C %mm4
	C %mm5
	C %mm6	h.low, then carry
	C %mm7	m replicated 4 times

	movd	%mm6, %ebx		C h.low
	psrlq	$32, %mm1		C l.high

	movd	%mm0, %edx		C h.high
	movq	%mm0, %mm6		C new c

	adcl	%eax, %ebx
	incl	%ecx

	movd	%mm1, %eax		C l.high
	movq	%mm7, %mm0

	adcl	%eax, %edx
	movl	%ebx, -16(%edi,%ecx,4)

	movl	%edx, -12(%edi,%ecx,4)
	psrlq	$32, %mm6		C c

L(small_entry):
	pmulhw	-8(%esi,%ecx,4), %mm0	C h = (x*m).high
	movq	%mm7, %mm1

	pmullw	-8(%esi,%ecx,4), %mm1	C l = (x*m).low
	movq	%mm7, %mm3

	movq	-8(%esi,%ecx,4), %mm2	C x
	psraw	$15, %mm3		C m signs

	pand	-8(%esi,%ecx,4), %mm3	C x selected by m signs
	psraw	$15, %mm2		C x signs

	paddw	%mm3, %mm0		C add x to h if m neg
	pand	%mm7, %mm2		C m selected by x signs

	paddw	%mm2, %mm0		C add m to h if x neg
	incl	%ecx

	movd	%mm1, %eax		C l.low
	punpcklwd %mm0, %mm6		C c + h.low << 16

	psrlq	$16, %mm0		C h.high
	js	L(small_top)




	movd	%mm6, %ebx		C h.low
	psrlq	$32, %mm1		C l.high

	adcl	%eax, %ebx
	popl	%ebp		FRAME_popl()

	movd	%mm0, %edx		C h.high
	psrlq	$32, %mm0		C l.high

	movd	%mm1, %eax		C l.high

	adcl	%eax, %edx
	movl	%ebx, -12(%edi,%ecx,4)

	movd	%mm0, %eax		C c

	adcl	$0, %eax
	movl	%edx, -8(%edi,%ecx,4)

	orl	%ecx, %ecx
	jnz	L(small_done)		C final %ecx==1 means even, ==0 odd


	C Size odd, one extra limb to process.
	C Plain integer code is used here, since it's smaller and is about
	C the same speed as another mmx block would be.

	movl	%eax, %ecx
	movl	PARAM_MULTIPLIER, %eax

	mull	-4(%esi)

	addl	%ecx, %eax

	adcl	$0, %edx
	movl	%eax, -4(%edi)

	movl	%edx, %eax
L(small_done):
	popl	%ebx

	popl	%edi
	popl	%esi

	emms

	ret

EPILOGUE()