diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/Makefile linux-2.4.0-test9-perfpatch/arch/alpha/lib/Makefile --- linux-2.4.0-test9.ref/arch/alpha/lib/Makefile Mon Jun 19 20:59:33 2000 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/Makefile Fri Oct 13 15:32:00 2000 @@ -3,9 +3,9 @@ # .S.s: - $(CC) -D__ASSEMBLY__ $(AFLAGS) -E -o $*.s $< + $(CC) -D__ASSEMBLY__ $(AFLAGS) $(CFLAGS) -E -o $*.s $< .S.o: - $(CC) -D__ASSEMBLY__ $(AFLAGS) -c -o $*.o $< + $(CC) -D__ASSEMBLY__ $(AFLAGS) $(CFLAGS) -c -o $*.o $< OBJS = __divqu.o __remqu.o __divlu.o __remlu.o memset.o memcpy.o io.o \ checksum.o csum_partial_copy.o strlen.o \ @@ -19,16 +19,16 @@ $(AR) rcs lib.a $(OBJS) __divqu.o: divide.S - $(CC) $(AFLAGS) -DDIV -c -o __divqu.o divide.S + $(CC) $(AFLAGS) $(CFLAGS) -DDIV -c -o __divqu.o divide.S __remqu.o: divide.S - $(CC) $(AFLAGS) -DREM -c -o __remqu.o divide.S + $(CC) $(AFLAGS) $(CFLAGS) -DREM -c -o __remqu.o divide.S __divlu.o: divide.S - $(CC) $(AFLAGS) -DDIV -DINTSIZE -c -o __divlu.o divide.S + $(CC) $(AFLAGS) $(CFLAGS) -DDIV -DINTSIZE -c -o __divlu.o divide.S __remlu.o: divide.S - $(CC) $(AFLAGS) -DREM -DINTSIZE -c -o __remlu.o divide.S + $(CC) $(AFLAGS) $(CFLAGS) -DREM -DINTSIZE -c -o __remlu.o divide.S dep: diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/checksum.c linux-2.4.0-test9-perfpatch/arch/alpha/lib/checksum.c --- linux-2.4.0-test9.ref/arch/alpha/lib/checksum.c Wed Apr 29 01:28:10 1998 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/checksum.c Mon Oct 9 10:52:54 2000 @@ -3,6 +3,10 @@ * * This file contains network checksum routines that are better done * in an architecture-specific manner due to speed.. + * Comments in other versions indicate that the algorithms are from RFC1071 + * + * accellerated versions (and 21264 assembly versions ) contributed by + * Rick Gorton */ #include @@ -11,15 +15,27 @@ static inline unsigned short from64to16(unsigned long x) { - /* add up 32-bit words for 33 bits */ - x = (x & 0xffffffff) + (x >> 32); - /* add up 16-bit and 17-bit words for 17+c bits */ - x = (x & 0xffff) + (x >> 16); - /* add up 16-bit and 2-bit for 16+c bit */ - x = (x & 0xffff) + (x >> 16); - /* add up carry.. */ - x = (x & 0xffff) + (x >> 16); - return x; + /* + * using extract instructions is a bit more efficient + * than the original shift/bitmask version. + */ + + union { + unsigned long ul; + unsigned int ui[2]; + unsigned short us[4]; + } in_v, tmp_v, out_v; + + in_v.ul = x; + tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1]; + out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1] + + (unsigned long) tmp_v.us[2]; + /* + * Since the bits of tmp_v.sh[3] are going to always be zero, + * we don't have to bother to add that in. + * Similarly, out_v.us[2] is always zero for the final add. + */ + return (out_v.us[0] + out_v.us[1]); } /* diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/clear_user.S linux-2.4.0-test9-perfpatch/arch/alpha/lib/clear_user.S --- linux-2.4.0-test9.ref/arch/alpha/lib/clear_user.S Sat Apr 24 20:54:08 1999 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/clear_user.S Mon Oct 9 10:52:54 2000 @@ -1,6 +1,7 @@ /* * arch/alpha/lib/clear_user.S * Contributed by Richard Henderson + * 21264 code contributed by Rick Gorton * * Zero user space, handling exceptions as we go. * @@ -33,6 +34,7 @@ lda $31, $exception-99b($31); \ .previous +#ifndef __alpha_ev6__ .set noat .set noreorder .align 4 @@ -112,3 +114,201 @@ ret $31, ($28), 1 # .. e1 : .end __do_clear_user + +#else +/* + * Much of the information about 21264 scheduling/coding comes from: + * Compiler Writer's Guide for the Alpha 21264 + * abbreviated as 'CWG' in other comments here + * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html + * Scheduling notation: + * E - either cluster + * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 + * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 + * Try not to change the actual algorithm if possible for consistency. + * Determining actual stalls (other than slotting) doesn't appear to be easy to do. + * From perusing the source code context where this routine is called, it is + * a fair assumption that significant fractions of entire pages are zeroed, so + * it's going to be worth the effort to hand-unroll a big loop, and use wh64. + * ASSUMPTION: + * The believed purpose of only updating $0 after a store is that a signal + * may come along during the execution of this chunk of code, and we don't + * want to leave a hole (and we also want to avoid repeating lots of work) + */ + + .set noat + .set noreorder + .align 4 + + .globl __do_clear_user + .ent __do_clear_user + .frame $30, 0, $28 + .prologue 0 + + # Pipeline info : Slotting & Comments +__do_clear_user: + ldgp $29,0($27) # we do exceptions -- we need the gp. + # Macro instruction becomes ldah/lda + # .. .. E E : + and $6, 7, $4 # .. E .. .. : find dest head misalignment + beq $0, $zerolength # U .. .. .. : U L U L + + addq $0, $4, $1 # .. .. .. E : bias counter + and $1, 7, $2 # .. .. E .. : number of misaligned bytes in tail +# Note - we never actually use $2, so this is a moot computation +# and we can rewrite this later... + srl $1, 3, $1 # .. E .. .. : number of quadwords to clear + beq $4, $headalign # U .. .. .. : U L U L + +/* + * Head is not aligned. Write (8 - $4) bytes to head of destination + * This means $6 is known to be misaligned + */ + EX( ldq_u $5, 0($6) ) # .. .. .. L : load dst word to mask back in + beq $1, $onebyte # .. .. U .. : sub-word store? + mskql $5, $6, $5 # .. U .. .. : take care of misaligned head + addq $6, 8, $6 # E .. .. .. : L U U L + + EX( stq_u $5, -8($6) ) # .. .. .. L : + subq $1, 1, $1 # .. .. E .. : + addq $0, $4, $0 # .. E .. .. : bytes left -= 8 - misalignment + subq $0, 8, $0 # E .. .. .. : U L U L + + .align 4 +/* + * (The .align directive ought to be a moot point) + * values upon initial entry to the loop + * $1 is number of quadwords to clear (zero is a valid value) + * $2 is number of trailing bytes (0..7) ($2 never used...) + * $6 is known to be aligned 0mod8 + */ +$headalign: + subq $1, 16, $4 # .. .. .. E : If < 16, we can not use the huge loop + and $6, 0x3f, $2 # .. .. E .. : Forward work for huge loop + subq $2, 0x40, $3 # .. E .. .. : bias counter (huge loop) + blt $4, $trailquad # U .. .. .. : U L U L + +/* + * We know that we're going to do at least 16 quads, which means we are + * going to be able to use the large block clear loop at least once. + * Figure out how many quads we need to clear before we are 0mod64 aligned + * so we can use the wh64 instruction. + */ + + nop # .. .. .. E + nop # .. .. E .. + nop # .. E .. .. + beq $3, $bigalign # U .. .. .. : U L U L : Aligned 0mod64 + +$alignmod64: + EX( stq_u $31, 0($6) ) # .. .. .. L + addq $3, 8, $3 # .. .. E .. + subq $0, 8, $0 # .. E .. .. + nop # E .. .. .. : U L U L + + nop # .. .. .. E + subq $1, 1, $1 # .. .. E .. + addq $6, 8, $6 # .. E .. .. + blt $3, $alignmod64 # U .. .. .. : U L U L + +$bigalign: +/* + * $0 is the number of bytes left + * $1 is the number of quads left + * $6 is aligned 0mod64 + * we know that we'll be taking a minimum of one trip through + * CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle + * We are _not_ going to update $0 after every single store. That + * would be silly, because there will be cross-cluster dependencies + * no matter how the code is scheduled. By doing it in slightly + * staggered fashion, we can still do this loop in 5 fetches + * The worse case will be doing two extra quads in some future execution, + * in the event of an interrupted clear. + * Assumes the wh64 needs to be for 2 trips through the loop in the future + * The wh64 is issued on for the starting destination address for trip +2 + * through the loop, and if there are less than two trips left, the target + * address will be for the current trip. + */ + nop # E : + nop # E : + nop # E : + bis $6,$6,$3 # E : U L U L : Initial wh64 address is dest + /* This might actually help for the current trip... */ + +$do_wh64: + wh64 ($3) # .. .. .. L1 : memory subsystem hint + subq $1, 16, $4 # .. .. E .. : Forward calculation - repeat the loop? + EX( stq_u $31, 0($6) ) # .. L .. .. + subq $0, 8, $0 # E .. .. .. : U L U L + + addq $6, 128, $3 # E : Target address of wh64 + EX( stq_u $31, 8($6) ) # L : + EX( stq_u $31, 16($6) ) # L : + subq $0, 16, $0 # E : U L L U + + nop # E : + EX( stq_u $31, 24($6) ) # L : + EX( stq_u $31, 32($6) ) # L : + subq $0, 168, $5 # E : U L L U : two trips through the loop left? + /* 168 = 192 - 24, since we've already completed some stores */ + + subq $0, 16, $0 # E : + EX( stq_u $31, 40($6) ) # L : + EX( stq_u $31, 48($6) ) # L : + cmovlt $5, $6, $3 # E : U L L U : Latency 2, extra mapping cycle + + subq $1, 8, $1 # E : + subq $0, 16, $0 # E : + EX( stq_u $31, 56($6) ) # L : + nop # E : U L U L + + nop # E : + subq $0, 8, $0 # E : + addq $6, 64, $6 # E : + bge $4, $do_wh64 # U : U L U L + +$trailquad: + # zero to 16 quadwords left to store, plus any trailing bytes + # $1 is the number of quadwords left to go. + # + nop # .. .. .. E + nop # .. .. E .. + nop # .. E .. .. + beq $1, $trailbytes # U .. .. .. : U L U L : Only 0..7 bytes to go + +$onequad: + EX( stq_u $31, 0($6) ) # .. .. .. L + subq $1, 1, $1 # .. .. E .. + subq $0, 8, $0 # .. E .. .. + nop # E .. .. .. : U L U L + + nop # .. .. .. E + nop # .. .. E .. + addq $6, 8, $6 # .. E .. .. + bgt $1, $onequad # U .. .. .. : U L U L + + # We have an unknown number of bytes left to go. +$trailbytes: + nop # .. .. .. E + nop # .. .. E .. + nop # .. E .. .. + beq $0, $zerolength # U .. .. .. : U L U L + + # $0 contains the number of bytes left to copy (0..31) + # so we will use $0 as the loop counter + # We know for a fact that $0 > 0 zero due to previous context +$onebyte: + EX( stb $31, 0($6) ) # .. .. .. L + subq $0, 1, $0 # .. .. E .. : + addq $6, 1, $6 # .. E .. .. : + bgt $0, $onebyte # U .. .. .. : U L U L + +$zerolength: +$exception: # Destination for exception recovery(?) + nop # .. .. .. E : + nop # .. .. E .. : + nop # .. E .. .. : + ret $31, ($28), 1 # L0 .. .. .. : L U L U + .end __do_clear_user + +#endif diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/copy_user.S linux-2.4.0-test9-perfpatch/arch/alpha/lib/copy_user.S --- linux-2.4.0-test9.ref/arch/alpha/lib/copy_user.S Fri Apr 28 01:27:16 2000 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/copy_user.S Mon Oct 9 10:52:54 2000 @@ -1,6 +1,9 @@ /* * arch/alpha/lib/copy_user.S * + * Contributed by ???? + * 21264 code contributed by Rick Gorton + * * Copy to/from user space, handling exceptions as we go.. This * isn't exactly pretty. * @@ -41,6 +44,7 @@ lda $31, $exitout-99b($31); \ .previous +#ifndef __alpha_ev6__ .set noat .align 3 .globl __copy_user @@ -144,3 +148,225 @@ ret $31,($28),1 .end __copy_user + +#else +/* + * Much of the information about 21264 scheduling/coding comes from: + * Compiler Writer's Guide for the Alpha 21264 + * abbreviated as 'CWG' in other comments here + * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html + * Scheduling notation: + * E - either cluster + * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 + * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 + */ + + .set noat + .align 4 + .globl __copy_user + .ent __copy_user + # Pipeline info: Slotting & Comments +__copy_user: + ldgp $29,0($27) # we do exceptions -- we need the gp. + # Macro instruction becomes ldah/lda + # .. .. E E + .prologue 1 + subq $0, 32, $1 # .. E .. .. : Is this going to be a small copy? + beq $0, $zerolength # U .. .. .. : U L U L + + and $6,7,$3 # .. .. .. E : is leading dest misalignment + ble $1, $onebyteloop # .. .. U .. : 1st branch : small amount of data + beq $3, $destaligned # .. U .. .. : 2nd (one cycle fetcher stall) + subq $3, 8, $3 # E .. .. .. : L U U L : trip counter +/* + * The fetcher stall also hides the 1 cycle cross-cluster stall for $3 (L --> U) + * This loop aligns the destination a byte at a time + * We know we have at least one trip through this loop + */ +$aligndest: + EXI( ldbu $1,0($7) ) # .. .. .. L : Keep loads separate from stores + addq $6,1,$6 # .. .. E .. : Section 3.8 in the CWG + addq $3,1,$3 # .. E .. .. : + nop # E .. .. .. : U L U L + +/* + * the -1 is to compensate for the inc($6) done in a previous quadpack + * which allows us zero dependencies within either quadpack in the loop + */ + EXO( stb $1,-1($6) ) # .. .. .. L : + addq $7,1,$7 # .. .. E .. : Section 3.8 in the CWG + subq $0,1,$0 # .. E .. .. : + bne $3, $aligndest # U .. .. .. : U L U L + +/* + * If we fell through into here, we have a minimum of 33 - 7 bytes + * If we arrived via branch, we have a minimum of 32 bytes + */ +$destaligned: + and $7,7,$1 # .. .. .. E : Check _current_ source alignment + bic $0,7,$4 # .. .. E .. : number bytes as a quadword loop + EXI( ldq_u $3,0($7) ) # .. L .. .. : Forward fetch for fallthrough code + beq $1,$quadaligned # U .. .. .. : U L U L + +/* + * In the worst case, we've just executed an ldq_u here from 0($7) + * and we'll repeat it once if we take the branch + */ + +/* Misaligned quadword loop - not unrolled. Leave it that way. */ +$misquad: + EXI( ldq_u $2,8($7) ) # .. .. .. L : + subq $4,8,$4 # .. .. E .. : + extql $3,$7,$3 # .. U .. .. : + extqh $2,$7,$1 # U .. .. .. : U U L L + + bis $3,$1,$1 # .. .. .. E : + EXO( stq $1,0($6) ) # .. .. L .. : + addq $7,8,$7 # .. E .. .. : + subq $0,8,$0 # E .. .. .. : U L L U + + addq $6,8,$6 # .. .. .. E : + bis $2,$2,$3 # .. .. E .. : + nop # .. E .. .. : + bne $4,$misquad # U .. .. .. : U L U L + + nop # .. .. .. E + nop # .. .. E .. + nop # .. E .. .. + beq $0,$zerolength # U .. .. .. : U L U L + +/* We know we have at least one trip through the byte loop */ + EXI ( ldbu $2,0($7) ) # .. .. .. L : No loads in the same quad + addq $6,1,$6 # .. .. E .. : as the store (Section 3.8 in CWG) + nop # .. E .. .. : + br $31, $dirtyentry # L0 .. .. .. : L U U L +/* Do the trailing byte loop load, then hop into the store part of the loop */ + +/* + * A minimum of (33 - 7) bytes to do a quad at a time. + * Based upon the usage context, it's worth the effort to unroll this loop + * $0 - number of bytes to be moved + * $4 - number of bytes to move as quadwords + * $6 is current destination address + * $7 is current source address + */ +$quadaligned: + subq $4, 32, $2 # .. .. .. E : do not unroll for small stuff + nop # .. .. E .. + nop # .. E .. .. + blt $2, $onequad # U .. .. .. : U L U L + +/* + * There is a significant assumption here that the source and destination + * addresses differ by more than 32 bytes. In this particular case, a + * sparsity of registers further bounds this to be a minimum of 8 bytes. + * But if this isn't met, then the output result will be incorrect. + * Furthermore, due to a lack of available registers, we really can't + * unroll this to be an 8x loop (which would enable us to use the wh64 + * instruction memory hint instruction). + */ +$unroll4: + EXI( ldq $1,0($7) ) # .. .. .. L + EXI( ldq $2,8($7) ) # .. .. L .. + subq $4,32,$4 # .. E .. .. + nop # E .. .. .. : U U L L + + addq $7,16,$7 # .. .. .. E + EXO( stq $1,0($6) ) # .. .. L .. + EXO( stq $2,8($6) ) # .. L .. .. + subq $0,16,$0 # E .. .. .. : U L L U + + addq $6,16,$6 # .. .. .. E + EXI( ldq $1,0($7) ) # .. .. L .. + EXI( ldq $2,8($7) ) # .. L .. .. + subq $4, 32, $3 # E .. .. .. : U U L L : is there enough for another trip? + + EXO( stq $1,0($6) ) # .. .. .. L + EXO( stq $2,8($6) ) # .. .. L .. + subq $0,16,$0 # .. E .. .. + addq $7,16,$7 # E .. .. .. : U L L U + + nop # .. .. .. E + nop # .. .. E .. + addq $6,16,$6 # .. E .. .. + bgt $3,$unroll4 # U .. .. .. : U L U L + + nop + nop + nop + beq $4, $noquads + +$onequad: + EXI( ldq $1,0($7) ) + subq $4,8,$4 + addq $7,8,$7 + nop + + EXO( stq $1,0($6) ) + subq $0,8,$0 + addq $6,8,$6 + bne $4,$onequad + +$noquads: + nop + nop + nop + beq $0,$zerolength + +/* + * For small copies (or the tail of a larger copy), do a very simple byte loop. + * There's no point in doing a lot of complex alignment calculations to try to + * to quadword stuff for a small amount of data. + * $0 - remaining number of bytes left to copy + * $6 - current dest addr + * $7 - current source addr + */ + +$onebyteloop: + EXI ( ldbu $2,0($7) ) # .. .. .. L : No loads in the same quad + addq $6,1,$6 # .. .. E .. : as the store (Section 3.8 in CWG) + nop # .. E .. .. : + nop # E .. .. .. : U L U L + +$dirtyentry: +/* + * the -1 is to compensate for the inc($6) done in a previous quadpack + * which allows us zero dependencies within either quadpack in the loop + */ + EXO ( stb $2,-1($6) ) # .. .. .. L : + addq $7,1,$7 # .. .. E .. : quadpack as the load + subq $0,1,$0 # .. E .. .. : change count _after_ copy + bgt $0,$onebyteloop # U .. .. .. : U L U L + +$zerolength: +$exitout: # Destination for exception recovery(?) + nop # .. .. .. E + nop # .. .. E .. + nop # .. E .. .. + ret $31,($28),1 # L0 .. .. .. : L U L U + +$exitin: + + /* A stupid byte-by-byte zeroing of the rest of the output + buffer. This cures security holes by never leaving + random kernel data around to be copied elsewhere. */ + + nop + nop + nop + mov $0,$1 + +$101: + EXO ( stb $31,0($6) ) # L + subq $1,1,$1 # E + addq $6,1,$6 # E + bgt $1,$101 # U + + nop + nop + nop + ret $31,($28),1 # L0 + + .end __copy_user + +#endif diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/csum_ipv6_magic.S linux-2.4.0-test9-perfpatch/arch/alpha/lib/csum_ipv6_magic.S --- linux-2.4.0-test9.ref/arch/alpha/lib/csum_ipv6_magic.S Thu Feb 17 12:18:46 2000 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/csum_ipv6_magic.S Mon Oct 9 10:52:54 2000 @@ -1,6 +1,7 @@ /* * arch/alpha/lib/csum_ipv6_magic.S * Contributed by Richard Henderson + * 21264 version contributed by Rick Gorton * * unsigned short csum_ipv6_magic(struct in6_addr *saddr, * struct in6_addr *daddr, @@ -9,6 +10,7 @@ * unsigned int csum); */ +#ifndef __alpha_ev6__ .globl csum_ipv6_magic .align 4 .ent csum_ipv6_magic @@ -90,3 +92,123 @@ ret # .. e1 : .end csum_ipv6_magic +#else + +/* + * Much of the information about 21264 scheduling/coding comes from: + * Compiler Writer's Guide for the Alpha 21264 + * abbreviated as 'CWG' in other comments here + * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html + * Scheduling notation: + * E - either cluster + * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 + * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 + * Try not to change the actual algorithm if possible for consistency. + * Determining actual stalls (other than slotting) doesn't appear to be easy to do. + * + * unsigned short csum_ipv6_magic(struct in6_addr *saddr, + * struct in6_addr *daddr, + * __u32 len, + * unsigned short proto, + * unsigned int csum); + * + * Swap (takes form 0xaabb) + * Then shift it left by 48, so result is: + * 0xbbaa0000 00000000 + * Then turn it back into a sign extended 32-bit item + * 0xbbaa0000 + * + * Swap (an unsigned int) using Mike Burrows' 7-instruction sequence + * (we can't hide the 3-cycle latency of the unpkbw in the 6-instruction sequence) + * Assume input takes form 0xAABBCCDD + * + * Finally, original 'folding' approach is to split the long into 4 unsigned shorts + * add 4 ushorts, resulting in ushort/carry + * add carry bits + ushort --> ushort + * add carry bits + ushort --> ushort (in case the carry results in an overflow) + * Truncate to a ushort. (took 13 instructions) + * From doing some testing, using the approach in checksum.c:from64to16() + * results in the same outcome: + * split into 2 uints, add those, generating a ulong + * add the 3 low ushorts together, generating a uint + * a final add of the 2 lower ushorts + * truncating the result. + */ + + .globl csum_ipv6_magic + .align 4 + .ent csum_ipv6_magic + .frame $30,0,$26,0 +csum_ipv6_magic: + .prologue 0 + + ldq $0,0($16) # L : Latency: 3 + inslh $18,7,$4 # U : 0000000000AABBCC + ldq $1,8($16) # L : Latency: 3 + sll $19,8,$7 # U : U L U L : 0x00000000 00aabb00 + + zapnot $20,15,$20 # U : zero extend incoming csum + ldq $2,0($17) # L : Latency: 3 + sll $19,24,$19 # U : U L L U : 0x000000aa bb000000 + inswl $18,3,$18 # U : 000000CCDD000000 + + ldq $3,8($17) # L : Latency: 3 + bis $18,$4,$18 # E : 000000CCDDAABBCC + addl $19,$7,$19 # E : bbaabb00 + nop # E : U L U L + + addq $20,$0,$20 # E : begin summing the words + srl $18,16,$4 # U : 0000000000CCDDAA + zap $19,0x3,$19 # U : bbaa0000 + nop # E : L U U L + + cmpult $20,$0,$0 # E : + addq $20,$1,$20 # E : + zapnot $18,0xa,$18 # U : 00000000DD00BB00 + zap $4,0xa,$4 # U : U U L L : 0000000000CC00AA + + or $18,$4,$18 # E : 00000000DDCCBBAA + nop # E : + cmpult $20,$1,$1 # E : + addq $20,$2,$20 # E : U L U L + + cmpult $20,$2,$2 # E : + addq $20,$3,$20 # E : + cmpult $20,$3,$3 # E : (1 cycle stall on $20) + addq $20,$18,$20 # E : U L U L (1 cycle stall on $20) + + cmpult $20,$18,$18 # E : + addq $20,$19,$20 # E : (1 cycle stall on $20) + addq $0,$1,$0 # E : merge the carries back into the csum + addq $2,$3,$2 # E : + + cmpult $20,$19,$19 # E : + addq $18,$19,$18 # E : (1 cycle stall on $19) + addq $0,$2,$0 # E : + addq $20,$18,$20 # E : U L U L : + /* (1 cycle stall on $18, 2 cycles on $20) */ + + addq $0,$20,$0 # E : + zapnot $0,15,$1 # U : Start folding output (1 cycle stall on $0) + nop # E : + srl $0,32,$0 # U : U L U L : (1 cycle stall on $0) + + addq $1,$0,$1 # E : Finished generating ulong + extwl $1,2,$2 # U : ushort[1] (1 cycle stall on $1) + zapnot $1,3,$0 # U : ushort[0] (1 cycle stall on $1) + extwl $1,4,$1 # U : ushort[2] (1 cycle stall on $1) + + addq $0,$2,$0 # E + addq $0,$1,$3 # E : Finished generating uint + /* (1 cycle stall on $0) */ + extwl $3,2,$1 # U : ushort[1] (1 cycle stall on $3) + nop # E : L U L U + + addq $1,$3,$0 # E : Final carry + not $0,$4 # E : complement (1 cycle stall on $0) + zapnot $4,3,$0 # U : clear upper garbage bits + /* (1 cycle stall on $4) */ + ret # L0 : L U L U + + .end csum_ipv6_magic +#endif diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/csum_partial_copy.c linux-2.4.0-test9-perfpatch/arch/alpha/lib/csum_partial_copy.c --- linux-2.4.0-test9.ref/arch/alpha/lib/csum_partial_copy.c Wed Jul 26 12:09:32 2000 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/csum_partial_copy.c Mon Oct 9 10:52:54 2000 @@ -2,6 +2,8 @@ * csum_partial_copy - do IP checksumming and copy * * (C) Copyright 1996 Linus Torvalds + * accellerated versions (and 21264 assembly versions ) contributed by + * Rick Gorton * * Don't look at this too closely - you'll go mad. The things * we do for performance.. @@ -68,6 +70,34 @@ }) +static inline unsigned short from64to16(unsigned long x) +{ + /* + * using extract instructions is a bit more efficient + * than the original shift/bitmask version. Cloned from + * checksum.c version. + */ + + union { + unsigned long ul; + unsigned int ui[2]; + unsigned short us[4]; + } in_v, tmp_v, out_v; + + in_v.ul = x; + tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1]; + out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1] + + (unsigned long) tmp_v.us[2]; + /* + * Since the bits of tmp_v.sh[3] are going to always be zero, + * we don't have to bother to add that in. + * Similarly, out_v.us[2] is always zero for the final add. + */ + return (out_v.us[0] + out_v.us[1]); +} + + + /* * Ok. This isn't fun, but this is the EASY case. */ @@ -335,13 +365,7 @@ soff, doff, len-8, checksum, partial_dest, errp); } - /* 64 -> 33 bits */ - checksum = (checksum & 0xffffffff) + (checksum >> 32); - /* 33 -> < 32 bits */ - checksum = (checksum & 0xffff) + (checksum >> 16); - /* 32 -> 16 bits */ - checksum = (checksum & 0xffff) + (checksum >> 16); - checksum = (checksum & 0xffff) + (checksum >> 16); + checksum = from64to16 (checksum); } return checksum; } diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/divide.S linux-2.4.0-test9-perfpatch/arch/alpha/lib/divide.S --- linux-2.4.0-test9.ref/arch/alpha/lib/divide.S Mon Oct 14 04:51:59 1996 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/divide.S Mon Oct 9 10:52:54 2000 @@ -2,6 +2,7 @@ * arch/alpha/lib/divide.S * * (C) 1995 Linus Torvalds + * 21264 version contributed by Rick Gorton * * Alpha division.. */ @@ -89,6 +90,7 @@ #define SLONGIFY(x) #endif +#ifndef __alpha_ev6__ .set noat .align 3 .globl ufunction @@ -193,3 +195,177 @@ addq $30,STACK,$30 ret $31,($23),1 .end sfunction + +#else + +/* + * Much of the information about 21264 scheduling/coding comes from: + * Compiler Writer's Guide for the Alpha 21264 + * abbreviated as 'CWG' in other comments here + * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html + * Scheduling notation: + * E - either cluster + * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 + * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 + * Try not to change the actual algorithm if possible for consistency. + */ + +.set noat +.align 4 +.globl ufunction +.ent ufunction +ufunction: + subq $30,STACK,$30 # E : + .frame $30,STACK,$23 + .prologue 0 + +7: stq $1, 0($30) # L : + bis $25,$25,divisor # E : + stq $2, 8($30) # L : L U L U + + bis $24,$24,modulus # E : + stq $0,16($30) # L : + bis $31,$31,quotient # E : + LONGIFY(divisor) # E : U L L U + + stq tmp1,24($30) # L : + LONGIFY(modulus) # E : + bis $31,1,mask # E : + DIV_ONLY(stq tmp2,32($30)) # L : L U U L + + beq divisor, 9f /* div by zero */ + /* + * In spite of the DIV_ONLY being either a non-instruction + * or an actual stq, the addition of the .align directive + * below ensures that label 1 is going to be nicely aligned + */ + + .align 4 +#ifdef INTSIZE + /* + * shift divisor left, using 3-bit shifts for + * 32-bit divides as we can't overflow. Three-bit + * shifts will result in looping three times less + * here, but can result in two loops more later. + * Thus using a large shift isn't worth it (and + * s8add pairs better than a sll..) + */ +1: cmpult divisor,modulus,compare # E : + s8addq divisor,$31,divisor # E : + s8addq mask,$31,mask # E : + bne compare,1b # U : U L U L +#else +1: cmpult divisor,modulus,compare # E : + nop # E : + nop # E : + blt divisor, 2f # U : U L U L + + addq divisor,divisor,divisor # E : + addq mask,mask,mask # E : + unop # E : + bne compare,1b # U : U L U L +#endif + + /* ok, start to go right again.. */ +2: + /* + * Keep things nicely bundled... use a nop instead of not + * having an instruction for DIV_ONLY + */ +#ifdef DIV + DIV_ONLY(addq quotient,mask,tmp2) # E : +#else + nop # E : +#endif + srl mask,1,mask # U : + cmpule divisor,modulus,compare # E : + subq modulus,divisor,tmp1 # E : + +#ifdef DIV + DIV_ONLY(cmovne compare,tmp2,quotient) # E : Latency 2, extra map slot + nop # E : as part of the cmovne + srl divisor,1,divisor # U : + nop # E : L U L U + + nop # E : + cmovne compare,tmp1,modulus # E : Latency 2, extra map slot + nop # E : as part of the cmovne + bne mask,2b # U : U L U L +#else + srl divisor,1,divisor # U : + cmovne compare,tmp1,modulus # E : Latency 2, extra map slot + nop # E : as part of the cmovne + bne mask,2b # U : U L L U +#endif + +9: ldq $1, 0($30) # L : + ldq $2, 8($30) # L : + nop # E : + nop # E : U U L L + + ldq $0,16($30) # L : + ldq tmp1,24($30) # L : + nop # E : + nop # E : + +#ifdef DIV + DIV_ONLY(ldq tmp2,32($30)) # L : +#else + nop # E : +#endif + addq $30,STACK,$30 # E : + ret $31,($23),1 # L0 : L U U L + .end ufunction + +/* + * Uhh.. Ugly signed division. I'd rather not have it at all, but + * it's needed in some circumstances. There are different ways to + * handle this, really. This does: + * -a / b = a / -b = -(a / b) + * -a % b = -(a % b) + * a % -b = a % b + * which is probably not the best solution, but at least should + * have the property that (x/y)*y + (x%y) = x. + */ +.align 4 +.globl sfunction +.ent sfunction +sfunction: + subq $30,STACK,$30 # E : + .frame $30,STACK,$23 + .prologue 0 + bis $24,$25,$28 # E : + SLONGIFY($28) # E : + bge $28,7b # U : + + stq $24,0($30) # L : + subq $31,$24,$28 # E : + stq $25,8($30) # L : + nop # E : U L U L + + cmovlt $24,$28,$24 /* abs($24) */ # E : Latency 2, extra map slot + nop # E : as part of the cmov + stq $23,16($30) # L : + subq $31,$25,$28 # E : U L U L + + stq tmp1,24($30) # L : + cmovlt $25,$28,$25 /* abs($25) */ # E : Latency 2, extra map slot + nop # E : + bsr $23,ufunction # L0: L U L U + + ldq $24,0($30) # L : + ldq $25,8($30) # L : + GETSIGN($28) # E : + subq $31,$27,tmp1 # E : U U L L + + SLONGIFY($28) # E : + ldq $23,16($30) # L : + cmovlt $28,tmp1,$27 # E : Latency 2, extra map slot + nop # E : U L L U : as part of the cmov + + ldq tmp1,24($30) # L : + nop # E : as part of the cmov + addq $30,STACK,$30 # E : + ret $31,($23),1 # L0 : L U U L + .end sfunction +#endif diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/memchr.S linux-2.4.0-test9-perfpatch/arch/alpha/lib/memchr.S --- linux-2.4.0-test9.ref/arch/alpha/lib/memchr.S Fri Aug 6 13:41:47 1999 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/memchr.S Mon Oct 9 10:52:54 2000 @@ -1,6 +1,7 @@ /* Copyright (C) 1996 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by David Mosberger (davidm@cs.arizona.edu). + 21264 version contributed by Rick Gorton The GNU C Library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as @@ -32,6 +33,7 @@ - the third argument is an unsigned long */ +#ifndef __alpha_ev6__ .set noreorder .set noat @@ -162,3 +164,180 @@ ret # .. e1 : .end memchr + +#else + +/* + * Much of the information about 21264 scheduling/coding comes from: + * Compiler Writer's Guide for the Alpha 21264 + * abbreviated as 'CWG' in other comments here + * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html + * Scheduling notation: + * E - either cluster + * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 + * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 + * Try not to change the actual algorithm if possible for consistency. + */ + + .set noreorder + .set noat + + .align 4 + .globl memchr + .ent memchr +memchr: + .frame $30,0,$26,0 + .prologue 0 + + # Hack -- if someone passes in (size_t)-1, hoping to just + # search til the end of the address space, we will overflow + # below when we find the address of the last byte. Given + # that we will never have a 56-bit address space, cropping + # the length is the easiest way to avoid trouble. + zap $18, 0x80, $5 # U : Bound length + beq $18, $not_found # U : + ldq_u $1, 0($16) # L : load first quadword Latency=3 + and $17, 0xff, $17 # E : L L U U : 00000000000000ch + + insbl $17, 1, $2 # U : 000000000000ch00 + cmpult $18, 9, $4 # E : small (< 1 quad) string? + or $2, $17, $17 # E : 000000000000chch + lda $3, -1($31) # E : U L L U + + sll $17, 16, $2 # U : 00000000chch0000 + addq $16, $5, $5 # E : Max search address + or $2, $17, $17 # E : 00000000chchchch + sll $17, 32, $2 # U : U L L U : chchchch00000000 + + or $2, $17, $17 # E : chchchchchchchch + extql $1, $16, $7 # U : $7 is upper bits + beq $4, $first_quad # U : + ldq_u $6, -1($5) # L : L U U L : eight or less bytes to search Latency=3 + + extqh $6, $16, $6 # U : 2 cycle stall for $6 + mov $16, $0 # E : + nop # E : + or $7, $6, $1 # E : L U L U $1 = quadword starting at $16 + + # Deal with the case where at most 8 bytes remain to be searched + # in $1. E.g.: + # $18 = 6 + # $1 = ????c6c5c4c3c2c1 +$last_quad: + negq $18, $6 # E : + xor $17, $1, $1 # E : + srl $3, $6, $6 # U : $6 = mask of $18 bits set + cmpbge $31, $1, $2 # E : L U L U + + nop + nop + and $2, $6, $2 # E : + beq $2, $not_found # U : U L U L + +$found_it: +#ifdef __alpha_cix__ + /* + * Since we are guaranteed to have set one of the bits, we don't + * have to worry about coming back with a 0x40 out of cttz... + */ + cttz $2, $3 # U0 : + addq $0, $3, $0 # E : All done + nop # E : + ret # L0 : L U L U +#else + /* + * Slow and clunky. It can probably be improved. + * An exercise left for others. + */ + negq $2, $3 # E : + and $2, $3, $2 # E : + and $2, 0x0f, $1 # E : + addq $0, 4, $3 # E : + + cmoveq $1, $3, $0 # E : Latency 2, extra map cycle + nop # E : keep with cmov + and $2, 0x33, $1 # E : + addq $0, 2, $3 # E : U L U L : 2 cycle stall on $0 + + cmoveq $1, $3, $0 # E : Latency 2, extra map cycle + nop # E : keep with cmov + and $2, 0x55, $1 # E : + addq $0, 1, $3 # E : U L U L : 2 cycle stall on $0 + + cmoveq $1, $3, $0 # E : Latency 2, extra map cycle + nop + nop + ret # L0 : L U L U +#endif + + # Deal with the case where $18 > 8 bytes remain to be + # searched. $16 may not be aligned. + .align 4 +$first_quad: + andnot $16, 0x7, $0 # E : + insqh $3, $16, $2 # U : $2 = 0000ffffffffffff ($16<0:2> ff) + xor $1, $17, $1 # E : + or $1, $2, $1 # E : U L U L $1 = ====ffffffffffff + + cmpbge $31, $1, $2 # E : + bne $2, $found_it # U : + # At least one byte left to process. + ldq $1, 8($0) # L : + subq $5, 1, $18 # E : U L U L + + addq $0, 8, $0 # E : + # Make $18 point to last quad to be accessed (the + # last quad may or may not be partial). + andnot $18, 0x7, $18 # E : + cmpult $0, $18, $2 # E : + beq $2, $final # U : U L U L + + # At least two quads remain to be accessed. + + subq $18, $0, $4 # E : $4 <- nr quads to be processed + and $4, 8, $4 # E : odd number of quads? + bne $4, $odd_quad_count # U : + # At least three quads remain to be accessed + mov $1, $4 # E : L U L U : move prefetched value to correct reg + + .align 4 +$unrolled_loop: + ldq $1, 8($0) # L : prefetch $1 + xor $17, $4, $2 # E : + cmpbge $31, $2, $2 # E : + bne $2, $found_it # U : U L U L + + addq $0, 8, $0 # E : + nop # E : + nop # E : + nop # E : + +$odd_quad_count: + xor $17, $1, $2 # E : + ldq $4, 8($0) # L : prefetch $4 + cmpbge $31, $2, $2 # E : + addq $0, 8, $6 # E : + + bne $2, $found_it # U : + cmpult $6, $18, $6 # E : + addq $0, 8, $0 # E : + nop # E : + + bne $6, $unrolled_loop # U : + mov $4, $1 # E : move prefetched value into $1 + nop # E : + nop # E : + +$final: subq $5, $0, $18 # E : $18 <- number of bytes left to do + nop # E : + nop # E : + bne $18, $last_quad # U : + +$not_found: + mov $31, $0 # E : + nop # E : + nop # E : + ret # L0 : + + .end memchr +#endif diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/memcpy.c linux-2.4.0-test9-perfpatch/arch/alpha/lib/memcpy.c --- linux-2.4.0-test9.ref/arch/alpha/lib/memcpy.c Mon Jun 7 14:37:13 1999 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/memcpy.c Mon Oct 9 10:52:54 2000 @@ -2,6 +2,7 @@ * linux/arch/alpha/lib/memcpy.c * * Copyright (C) 1995 Linus Torvalds + * 21264 version contributed by Rick Gorton */ /* @@ -110,6 +111,9 @@ * for the load-store. I don't know why, but it would seem that using a floating * point register for the move seems to slow things down (very small difference, * though). + * RGorton note: Forcing integer registers is a 'good thing'. Some + * implementations of Alpha (ev6) have float loads/stores with longer + * latencies than for integer loads/stores. * * Note the ordering to try to avoid load (and address generation) latencies. */ @@ -117,18 +121,149 @@ long n) { ALIGN_DEST_TO8_UP(d,s,n); +#ifdef __alpha_ev6__ + if (n >= 128) { + unsigned long wh64_addr; + /* + * Lots to move. First, get 0mod64 aligned, then + * traverse an unrolled 64-byte loop. Use wh64 to + * speed the writing. + * future enhancement: use prefetch instructions for source + */ + while (d & 63) { + unsigned long val1_t; + /* Quadpack */ + __asm__ __volatile__ ("ldq %0,%1":"=r" (val1_t):"m" (*(unsigned long *) s)); + n -= 8; + s += 8; + __asm__ __volatile__ ("nop"); + + /* Quadpack */ + __asm__ __volatile__ ("stq %0,%1":"=r" (val1_t):"m" (*(unsigned long *) d)); + d += 8; + /* Control flow is two instructions to fill the quadpack */ + } + /* + * Got to do this addq as an asm statement to force syntax: + * wh64 (reg) Notice lack of 0 offset, even + * though it is a memory address + * If the assembler changes to accept a zero offset on wh64, + * this hack wouldn't have to be an asm(). + */ + __asm__ __volatile__ ("bis %0,%1,%2":"=r" (d):"r" (d),"r" (wh64_addr)); + while (n >= 64) + { + unsigned long val1_t, val2_t, val3_t, val4_t; + unsigned long register twomore, next_d; + + /* Quadpack */ + __asm__ __volatile__ ("wh64 (%0)":"=r" (wh64_addr)); + __asm__ __volatile__ ("ldq %0,%1":"=r" (val1_t):"m" (*(unsigned long *) s)); + __asm__ __volatile__ ("nop"); + __asm__ __volatile__ ("nop"); + + /* Quadpack */ + __asm__ __volatile__ ("ldq %0,%1":"=r" (val2_t):"m" (*(unsigned long *) (s+8))); + __asm__ __volatile__ ("ldq %0,%1":"=r" (val3_t):"m" (*(unsigned long *) (s+16))); + wh64_addr +=64; + __asm__ __volatile__ ("nop"); + + /* Quadpack */ + + __asm__ __volatile__ ("ldq %0,%1":"=r" (val4_t):"m" (*(unsigned long *) (s+24))); + next_d = d + 64; + __asm__ __volatile__ ("nop"); + __asm__ __volatile__ ("nop"); + + /* Quadpack */ + *(unsigned long *)d = val1_t; + *(unsigned long *)(d+8) = val2_t; + s += 32; + __asm__ __volatile__ ("nop"); + + /* Quadpack */ + *(unsigned long *)(d+16) = val3_t; + *(unsigned long *)(d+24) = val4_t; + d += 32; + /* + * Again, a hack. Couldn't seem to get cmovlt + * to pick up the correct register unless this was + * done as an asm statement. Annoying. + */ + __asm__ __volatile__ ("subq %0,%1,%2" + :"=r" (n) + :"I" (128) + ,"r" (twomore)); + + /* Quadpack */ + __asm__ __volatile__ ("ldq %0,%1":"=r" (val1_t):"m" (*(unsigned long *) s)); + __asm__ __volatile__ ("ldq %0,%1":"=r" (val2_t):"m" (*(unsigned long *) (s+8))); + __asm__ __volatile__ ("cmovlt %0,%1,%2":"=r" (twomore):"r" (next_d),"r" (wh64_addr)); + __asm__ __volatile__ ("nop"); + + /* Quadpack */ + __asm__ __volatile__ ("ldq %0,%1":"=r" (val3_t):"m" (*(unsigned long *) (s+16))); + __asm__ __volatile__ ("ldq %0,%1":"=r" (val4_t):"m" (*(unsigned long *) (s+24))); + n -= 64; + d += 32; + + /* Quadpack */ + *(unsigned long *)(d-32) = val1_t; + *(unsigned long *)(d-24) = val2_t; + s += 32; + __asm__ __volatile__ ("nop"); + + /* Quadpack */ + *(unsigned long *)(d-16) = val3_t; + *(unsigned long *)(d-8) = val4_t; + /* + * After all that work to carefully schedule stuff, + * the loop control results in + * cmple a2,0x3f,t0 + * beq t0,top + * With both instructions in the same quadpack, resulting + * in a 1 cycle stall. Really quite annoying. + * gcc needs to get fixed. + */ + } + __asm__ __volatile__ (".align 4"); + } + __asm__ __volatile__ (".align 4"); +#endif n -= 8; while (n >= 0) { unsigned long tmp; __asm__("ldq %0,%1":"=r" (tmp):"m" (*(unsigned long *) s)); n -= 8; s += 8; + __asm__ __volatile__ ("nop"); + /* + * Results in a full quadpack for ev6, as well as correctly + * separating load quads from store quads For other Alpha + * implementations, prevents stalls due to load latency + */ *(unsigned long *) d = tmp; d += 8; } + __asm__ __volatile__ (".align 4"); n += 8; +#ifdef __alpha_ev6__ + while (n > 0) { + unsigned char ctmp; + n--; + __asm__ __volatile__ ("ldbu %0,%1":"=r" (ctmp):"m" (*(unsigned long *) s)); + s++; + __asm__ __volatile__ ("nop"); + + *(char *) d = ctmp; + d++; + } + __asm__ __volatile__ (".align 4"); +#else DO_REST_ALIGNED_UP(d,s,n); +#endif } + static inline void __memcpy_aligned_dn (unsigned long d, unsigned long s, long n) { @@ -153,9 +288,11 @@ if (!(((unsigned long) dest ^ (unsigned long) src) & 7)) { __memcpy_aligned_up ((unsigned long) dest, (unsigned long) src, n); + __asm__ __volatile__ (".align 4"); return dest; } __memcpy_unaligned_up ((unsigned long) dest, (unsigned long) src, n); + __asm__ __volatile__ (".align 4"); return dest; } @@ -180,5 +317,6 @@ __memcpy_unaligned_dn ((unsigned long) dest, (unsigned long) src, n); } + __asm__ __volatile__ (".align 4"); return dest; } diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/memset.S linux-2.4.0-test9-perfpatch/arch/alpha/lib/memset.S --- linux-2.4.0-test9.ref/arch/alpha/lib/memset.S Sun Jul 19 23:44:41 1998 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/memset.S Mon Oct 9 10:52:54 2000 @@ -5,6 +5,7 @@ * function for the alpha. * * (C) Copyright 1996 Linus Torvalds + * 21264 version contributed by Rick Gorton * * This routine is "moral-ware": you are free to use it any way you wish, and * the only obligation I put on you is a moral one: if you make any improvements @@ -20,6 +21,8 @@ .globl __memset .globl __memsetw .globl __constant_c_memset + +#ifndef __alpha_ev6__ .ent __memset .align 5 __memset: @@ -119,6 +122,591 @@ br __constant_c_memset /* .. E1 */ .end __memsetw + +#else + +/* + * Much of the information about 21264 scheduling/coding comes from: + * Compiler Writer's Guide for the Alpha 21264 + * abbreviated as 'CWG' in other comments here + * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html + * Scheduling notation: + * E - either cluster + * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 + * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 + * The algorithm for the leading and trailing quadwords remains the same, + * however the loop has been unrolled to enable better memory throughput, + * and the code has been replicated for each of the entry points: __memset + * and __memsetw to permit better scheduling to eliminate the stalling + * encountered during the mask replication. + * A future enhancement might be to put in a byte store loop for really + * small (say < 32 bytes) memset()s. Whether or not that change would be + * a win in the kernel would depend upon the contextual usage. + * WARNING: Maintaining this is going to be more work than the above version, + * as fixes will need to be made in multiple places. The performance gain + * is worth it. + */ + + + .ent __memset +.align 5 +__memset: + .frame $30,0,$26,0 + .prologue 0 + + /* + * Serious stalling happens. The only way to mitigate this is to + * undertake a major re-write to interleave the constant materialization + * with other parts of the fall-through code. This is important, even + * though it makes maintenance tougher. + * Do this later. + */ + and $17,255,$1 # E : 00000000000000ch + insbl $17,1,$2 # U : 000000000000ch00 + bis $16,$16,$0 # E : return value + ble $18,end_b # U : zero length requested? + + addq $18,$16,$6 # E : max address to write to + bis $1,$2,$17 # E : 000000000000chch + insbl $1,2,$3 # U : 0000000000ch0000 + insbl $1,3,$4 # U : 00000000ch000000 + + or $3,$4,$3 # E : 00000000chch0000 + inswl $17,4,$5 # U : 0000chch00000000 + xor $16,$6,$1 # E : will complete write be within one quadword? + inswl $17,6,$2 # U : chch000000000000 + + or $17,$3,$17 # E : 00000000chchchch + or $2,$5,$2 # E : chchchch00000000 + bic $1,7,$1 # E : fit within a single quadword? + and $16,7,$3 # E : Target addr misalignment + + or $17,$2,$17 # E : chchchchchchchch + beq $1,within_quad_b # U : + nop # E : + beq $3,aligned_b # U : target is 0mod8 + + /* + * Target address is misaligned, and won't fit within a quadword + */ + ldq_u $4,0($16) # L : Fetch first partial + bis $16,$16,$5 # E : Save the address + insql $17,$16,$2 # U : Insert new bytes + subq $3,8,$3 # E : Invert (for addressing uses) + + addq $18,$3,$18 # E : $18 is new count ($3 is negative) + mskql $4,$16,$4 # U : clear relevant parts of the quad + subq $16,$3,$16 # E : $16 is new aligned destination + bis $2,$4,$1 # E : Final bytes + + nop + stq_u $1,0($5) # L : Store result + nop + nop + +.align 4 +aligned_b: + /* + * We are now guaranteed to be quad aligned, with at least + * one partial quad to write. + */ + + sra $18,3,$3 # U : Number of remaining quads to write + and $18,7,$18 # E : Number of trailing bytes to write + bis $16,$16,$5 # E : Save dest address + beq $3,no_quad_b # U : tail stuff only + + /* + * it's worth the effort to unroll this and use wh64 if possible + * Lifted a bunch of code from clear_user.S + * At this point, entry values are: + * $16 Current destination address + * $5 A copy of $16 + * $6 The max quadword address to write to + * $18 Number trailer bytes + * $3 Number quads to write + */ + + and $16, 0x3f, $2 # E : Forward work (only useful for unrolled loop) + subq $3, 16, $4 # E : Only try to unroll if > 128 bytes + subq $2, 0x40, $1 # E : bias counter (aligning stuff 0mod64) + blt $4, loop_b # U : + + /* + * We know we've got at least 16 quads, minimum of one trip + * through unrolled loop. Do a quad at a time to get us 0mod64 + * aligned. + */ + + nop # E : + nop # E : + nop # E : + beq $1, $bigalign_b # U : + +$alignmod64_b: + stq $17, 0($5) # L : + subq $3, 1, $3 # E : For consistency later + addq $1, 8, $1 # E : Increment towards zero for alignment + addq $5, 8, $4 # E : Initial wh64 address (filler instruction) + + nop + nop + addq $5, 8, $5 # E : Inc address + blt $1, $alignmod64_b # U : + +$bigalign_b: + /* + * $3 - number quads left to go + * $5 - target address (aligned 0mod64) + * $17 - mask of stuff to store + * Scratch registers available: $7, $2, $4, $1 + * we know that we'll be taking a minimum of one trip through + * CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle + * Assumes the wh64 needs to be for 2 trips through the loop in the future + * The wh64 is issued on for the starting destination address for trip +2 + * through the loop, and if there are less than two trips left, the target + * address will be for the current trip. + */ + +$do_wh64_b: + wh64 ($4) # L1 : memory subsystem write hint + subq $3, 24, $2 # E : For determining future wh64 addresses + stq $17, 0($5) # L : + nop # E : + + addq $5, 128, $4 # E : speculative target of next wh64 + stq $17, 8($5) # L : + stq $17, 16($5) # L : + addq $5, 64, $7 # E : Fallback address for wh64 (== next trip addr) + + stq $17, 24($5) # L : + stq $17, 32($5) # L : + cmovlt $2, $7, $4 # E : Latency 2, extra mapping cycle + nop + + stq $17, 40($5) # L : + stq $17, 48($5) # L : + subq $3, 16, $2 # E : Repeat the loop at least once more? + nop + + stq $17, 56($5) # L : + addq $5, 64, $5 # E : + subq $3, 8, $3 # E : + bge $2, $do_wh64_b # U : + + nop + nop + nop + beq $3, no_quad_b # U : Might have finished already + +.align 4 + /* + * Simple loop for trailing quadwords, or for small amounts + * of data (where we can't use an unrolled loop and wh64) + */ +loop_b: + stq $17,0($5) # L : + subq $3,1,$3 # E : Decrement number quads left + addq $5,8,$5 # E : Inc address + bne $3,loop_b # U : more? + +no_quad_b: + /* + * Write 0..7 trailing bytes. + */ + nop # E : + beq $18,end_b # U : All done? + ldq $7,0($5) # L : + mskqh $7,$6,$2 # U : Mask final quad + + insqh $17,$6,$4 # U : New bits + bis $2,$4,$1 # E : Put it all together + stq $1,0($5) # L : And back to memory + ret $31,($26),1 # L0 : + +within_quad_b: + ldq_u $1,0($16) # L : + insql $17,$16,$2 # U : New bits + mskql $1,$16,$4 # U : Clear old + bis $2,$4,$2 # E : New result + + mskql $2,$6,$4 # U : + mskqh $1,$6,$2 # U : + bis $2,$4,$1 # E : + stq_u $1,0($16) # L : + +end_b: + nop + nop + nop + ret $31,($26),1 # L0 : + .end __memset + + /* + * This is the original body of code, prior to replication and + * rescheduling. Leave it here, as there may be calls to this + * entry point. + */ +.align 4 + .ent __memset +__constant_c_memset: + .frame $30,0,$26,0 + .prologue 0 + + addq $18,$16,$6 # E : max address to write to + bis $16,$16,$0 # E : return value + xor $16,$6,$1 # E : will complete write be within one quadword? + ble $18,end # U : zero length requested? + + bic $1,7,$1 # E : fit within a single quadword + beq $1,within_one_quad # U : + and $16,7,$3 # E : Target addr misalignment + beq $3,aligned # U : target is 0mod8 + + /* + * Target address is misaligned, and won't fit within a quadword + */ + ldq_u $4,0($16) # L : Fetch first partial + bis $16,$16,$5 # E : Save the address + insql $17,$16,$2 # U : Insert new bytes + subq $3,8,$3 # E : Invert (for addressing uses) + + addq $18,$3,$18 # E : $18 is new count ($3 is negative) + mskql $4,$16,$4 # U : clear relevant parts of the quad + subq $16,$3,$16 # E : $16 is new aligned destination + bis $2,$4,$1 # E : Final bytes + + nop + stq_u $1,0($5) # L : Store result + nop + nop + +.align 4 +aligned: + /* + * We are now guaranteed to be quad aligned, with at least + * one partial quad to write. + */ + + sra $18,3,$3 # U : Number of remaining quads to write + and $18,7,$18 # E : Number of trailing bytes to write + bis $16,$16,$5 # E : Save dest address + beq $3,no_quad # U : tail stuff only + + /* + * it's worth the effort to unroll this and use wh64 if possible + * Lifted a bunch of code from clear_user.S + * At this point, entry values are: + * $16 Current destination address + * $5 A copy of $16 + * $6 The max quadword address to write to + * $18 Number trailer bytes + * $3 Number quads to write + */ + + and $16, 0x3f, $2 # E : Forward work (only useful for unrolled loop) + subq $3, 16, $4 # E : Only try to unroll if > 128 bytes + subq $2, 0x40, $1 # E : bias counter (aligning stuff 0mod64) + blt $4, loop # U : + + /* + * We know we've got at least 16 quads, minimum of one trip + * through unrolled loop. Do a quad at a time to get us 0mod64 + * aligned. + */ + + nop # E : + nop # E : + nop # E : + beq $1, $bigalign # U : + +$alignmod64: + stq $17, 0($5) # L : + subq $3, 1, $3 # E : For consistency later + addq $1, 8, $1 # E : Increment towards zero for alignment + addq $5, 8, $4 # E : Initial wh64 address (filler instruction) + + nop + nop + addq $5, 8, $5 # E : Inc address + blt $1, $alignmod64 # U : + +$bigalign: + /* + * $3 - number quads left to go + * $5 - target address (aligned 0mod64) + * $17 - mask of stuff to store + * Scratch registers available: $7, $2, $4, $1 + * we know that we'll be taking a minimum of one trip through + * CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle + * Assumes the wh64 needs to be for 2 trips through the loop in the future + * The wh64 is issued on for the starting destination address for trip +2 + * through the loop, and if there are less than two trips left, the target + * address will be for the current trip. + */ + +$do_wh64: + wh64 ($4) # L1 : memory subsystem write hint + subq $3, 24, $2 # E : For determining future wh64 addresses + stq $17, 0($5) # L : + nop # E : + + addq $5, 128, $4 # E : speculative target of next wh64 + stq $17, 8($5) # L : + stq $17, 16($5) # L : + addq $5, 64, $7 # E : Fallback address for wh64 (== next trip addr) + + stq $17, 24($5) # L : + stq $17, 32($5) # L : + cmovlt $2, $7, $4 # E : Latency 2, extra mapping cycle + nop + + stq $17, 40($5) # L : + stq $17, 48($5) # L : + subq $3, 16, $2 # E : Repeat the loop at least once more? + nop + + stq $17, 56($5) # L : + addq $5, 64, $5 # E : + subq $3, 8, $3 # E : + bge $2, $do_wh64 # U : + + nop + nop + nop + beq $3, no_quad # U : Might have finished already + +.align 4 + /* + * Simple loop for trailing quadwords, or for small amounts + * of data (where we can't use an unrolled loop and wh64) + */ +loop: + stq $17,0($5) # L : + subq $3,1,$3 # E : Decrement number quads left + addq $5,8,$5 # E : Inc address + bne $3,loop # U : more? + +no_quad: + /* + * Write 0..7 trailing bytes. + */ + nop # E : + beq $18,end # U : All done? + ldq $7,0($5) # L : + mskqh $7,$6,$2 # U : Mask final quad + + insqh $17,$6,$4 # U : New bits + bis $2,$4,$1 # E : Put it all together + stq $1,0($5) # L : And back to memory + ret $31,($26),1 # L0 : + +within_one_quad: + ldq_u $1,0($16) # L : + insql $17,$16,$2 # U : New bits + mskql $1,$16,$4 # U : Clear old + bis $2,$4,$2 # E : New result + + mskql $2,$6,$4 # U : + mskqh $1,$6,$2 # U : + bis $2,$4,$1 # E : + stq_u $1,0($16) # L : + +end: + nop + nop + nop + ret $31,($26),1 # L0 : + .end __constant_c_memset + + /* + * This is a replicant of the __constant_c_memset code, rescheduled + * to mask stalls. Note that entry point names also had to change + */ + .align 5 + .ent __memsetw + +__memsetw: + .frame $30,0,$26,0 + .prologue 0 + + inswl $17,0,$5 # U : 000000000000c1c2 + inswl $17,2,$2 # U : 00000000c1c20000 + bis $16,$16,$0 # E : return value + addq $18,$16,$6 # E : max address to write to + + ble $18, end_w # U : zero length requested? + inswl $17,4,$3 # U : 0000c1c200000000 + inswl $17,6,$4 # U : c1c2000000000000 + xor $16,$6,$1 # E : will complete write be within one quadword? + + or $2,$5,$2 # E : 00000000c1c2c1c2 + or $3,$4,$17 # E : c1c2c1c200000000 + bic $1,7,$1 # E : fit within a single quadword + and $16,7,$3 # E : Target addr misalignment + + or $17,$2,$17 # E : c1c2c1c2c1c2c1c2 + beq $1,within_quad_w # U : + nop + beq $3,aligned_w # U : target is 0mod8 + + /* + * Target address is misaligned, and won't fit within a quadword + */ + ldq_u $4,0($16) # L : Fetch first partial + bis $16,$16,$5 # E : Save the address + insql $17,$16,$2 # U : Insert new bytes + subq $3,8,$3 # E : Invert (for addressing uses) + + addq $18,$3,$18 # E : $18 is new count ($3 is negative) + mskql $4,$16,$4 # U : clear relevant parts of the quad + subq $16,$3,$16 # E : $16 is new aligned destination + bis $2,$4,$1 # E : Final bytes + + nop + stq_u $1,0($5) # L : Store result + nop + nop + +.align 4 +aligned_w: + /* + * We are now guaranteed to be quad aligned, with at least + * one partial quad to write. + */ + + sra $18,3,$3 # U : Number of remaining quads to write + and $18,7,$18 # E : Number of trailing bytes to write + bis $16,$16,$5 # E : Save dest address + beq $3,no_quad_w # U : tail stuff only + + /* + * it's worth the effort to unroll this and use wh64 if possible + * Lifted a bunch of code from clear_user.S + * At this point, entry values are: + * $16 Current destination address + * $5 A copy of $16 + * $6 The max quadword address to write to + * $18 Number trailer bytes + * $3 Number quads to write + */ + + and $16, 0x3f, $2 # E : Forward work (only useful for unrolled loop) + subq $3, 16, $4 # E : Only try to unroll if > 128 bytes + subq $2, 0x40, $1 # E : bias counter (aligning stuff 0mod64) + blt $4, loop_w # U : + + /* + * We know we've got at least 16 quads, minimum of one trip + * through unrolled loop. Do a quad at a time to get us 0mod64 + * aligned. + */ + + nop # E : + nop # E : + nop # E : + beq $1, $bigalign_w # U : + +$alignmod64_w: + stq $17, 0($5) # L : + subq $3, 1, $3 # E : For consistency later + addq $1, 8, $1 # E : Increment towards zero for alignment + addq $5, 8, $4 # E : Initial wh64 address (filler instruction) + + nop + nop + addq $5, 8, $5 # E : Inc address + blt $1, $alignmod64_w # U : + +$bigalign_w: + /* + * $3 - number quads left to go + * $5 - target address (aligned 0mod64) + * $17 - mask of stuff to store + * Scratch registers available: $7, $2, $4, $1 + * we know that we'll be taking a minimum of one trip through + * CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle + * Assumes the wh64 needs to be for 2 trips through the loop in the future + * The wh64 is issued on for the starting destination address for trip +2 + * through the loop, and if there are less than two trips left, the target + * address will be for the current trip. + */ + +$do_wh64_w: + wh64 ($4) # L1 : memory subsystem write hint + subq $3, 24, $2 # E : For determining future wh64 addresses + stq $17, 0($5) # L : + nop # E : + + addq $5, 128, $4 # E : speculative target of next wh64 + stq $17, 8($5) # L : + stq $17, 16($5) # L : + addq $5, 64, $7 # E : Fallback address for wh64 (== next trip addr) + + stq $17, 24($5) # L : + stq $17, 32($5) # L : + cmovlt $2, $7, $4 # E : Latency 2, extra mapping cycle + nop + + stq $17, 40($5) # L : + stq $17, 48($5) # L : + subq $3, 16, $2 # E : Repeat the loop at least once more? + nop + + stq $17, 56($5) # L : + addq $5, 64, $5 # E : + subq $3, 8, $3 # E : + bge $2, $do_wh64_w # U : + + nop + nop + nop + beq $3, no_quad_w # U : Might have finished already + +.align 4 + /* + * Simple loop for trailing quadwords, or for small amounts + * of data (where we can't use an unrolled loop and wh64) + */ +loop_w: + stq $17,0($5) # L : + subq $3,1,$3 # E : Decrement number quads left + addq $5,8,$5 # E : Inc address + bne $3,loop_w # U : more? + +no_quad_w: + /* + * Write 0..7 trailing bytes. + */ + nop # E : + beq $18,end_w # U : All done? + ldq $7,0($5) # L : + mskqh $7,$6,$2 # U : Mask final quad + + insqh $17,$6,$4 # U : New bits + bis $2,$4,$1 # E : Put it all together + stq $1,0($5) # L : And back to memory + ret $31,($26),1 # L0 : + +within_quad_w: + ldq_u $1,0($16) # L : + insql $17,$16,$2 # U : New bits + mskql $1,$16,$4 # U : Clear old + bis $2,$4,$2 # E : New result + + mskql $2,$6,$4 # U : + mskqh $1,$6,$2 # U : + bis $2,$4,$1 # E : + stq_u $1,0($16) # L : + +end_w: + nop + nop + nop + ret $31,($26),1 # L0 : + + .end __memsetw +#endif #ifdef __ELF__ .weak memset; memset = __memset diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/strcasecmp.c linux-2.4.0-test9-perfpatch/arch/alpha/lib/strcasecmp.c --- linux-2.4.0-test9.ref/arch/alpha/lib/strcasecmp.c Sun Aug 9 15:09:05 1998 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/strcasecmp.c Mon Oct 9 10:52:54 2000 @@ -1,5 +1,7 @@ /* * linux/arch/alpha/lib/strcasecmp.c + * Original Author: ??? + * performance tuneup by Rick Gorton */ #include @@ -11,7 +13,13 @@ int strcasecmp(const char *a, const char *b) { - int ca, cb; + unsigned long ca, cb; + /* + * Originally, declaration was 'int ca, cb;'. Switching + * to unsigned long permits much cleaner code sequences, + * and avoids having to sign and zero extend values until + * the final operation. + */ do { ca = *a++ & 0xff; diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/strcat.S linux-2.4.0-test9-perfpatch/arch/alpha/lib/strcat.S --- linux-2.4.0-test9.ref/arch/alpha/lib/strcat.S Fri Oct 18 04:53:20 1996 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/strcat.S Thu Nov 9 16:36:19 2000 @@ -1,12 +1,62 @@ /* * arch/alpha/lib/strcat.S * Contributed by Richard Henderson (rth@tamu.edu) + * 21264 tuned version by Rick Gorton * * Append a null-terminated string from SRC to DST. */ .text +#if (defined(__alpha_fix__) || defined(__alpha_cix__)) && defined(CONFIG_ALPHA_EV67) + +/* + * Much of the information about 21264 scheduling/coding comes from: + * Compiler Writer's Guide for the Alpha 21264 + * abbreviated as 'CWG' in other comments here + * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html + * Scheduling notation: + * E - either cluster + * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 + * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 + * Try not to change the actual algorithm if possible for consistency. + * Commentary: It seems bogus to walk the input string twice - once + * to determine the length, and then again while doing the copy. + * A significant (future) enhancement would be to only read the input + * string once. + */ + + .align 4 + .globl strcat + .ent strcat +strcat: + .frame $30, 0, $26 + .prologue 0 + + mov $16, $0 # E : set up return value + /* Find the end of the string. */ + ldq_u $1, 0($16) # L : load first quadword (a0 may be misaligned) + lda $2, -1 # E : + insqh $2, $16, $2 # U : + + andnot $16, 7, $16 # E : + or $2, $1, $1 # E : + cmpbge $31, $1, $2 # E : bits set iff byte == 0 + bne $2, $found # U : + +$loop: ldq $1, 8($16) # L : + addq $16, 8, $16 # E : + cmpbge $31, $1, $2 # E : + beq $2, $loop # U : + +$found: cttz $2, $3 # U0 : + addq $16, $3, $16 # E : + /* Now do the append. */ + mov $26, $23 # E : + br __stxcpy # L0 : + + .end strcat +#else .align 3 .globl strcat .ent strcat @@ -50,3 +100,4 @@ br __stxcpy .end strcat +#endif diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/strchr.S linux-2.4.0-test9-perfpatch/arch/alpha/lib/strchr.S --- linux-2.4.0-test9.ref/arch/alpha/lib/strchr.S Fri Oct 18 04:53:20 1996 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/strchr.S Thu Nov 9 16:36:19 2000 @@ -1,6 +1,7 @@ /* * arch/alpha/lib/strchr.S * Contributed by Richard Henderson (rth@tamu.edu) + * 21264 version contributed by Rick Gorton * * Return the address of a given character within a null-terminated * string, or null if it is not found. @@ -8,6 +9,89 @@ #include +#if (defined(__alpha_fix__) || defined(__alpha_cix__)) && defined(CONFIG_ALPHA_EV67) + +/* + * Much of the information about 21264 scheduling/coding comes from: + * Compiler Writer's Guide for the Alpha 21264 + * abbreviated as 'CWG' in other comments here + * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html + * Scheduling notation: + * E - either cluster + * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 + * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 + * Try not to change the actual algorithm if possible for consistency. + */ + .set noreorder + .set noat + + .align 4 + .globl strchr + .ent strchr +strchr: + .frame sp, 0, ra + .prologue 0 + + ldq_u t0, 0(a0) # L : load first quadword Latency=3 + and a1, 0xff, t3 # E : 00000000000000ch + insbl a1, 1, t5 # U : 000000000000ch00 + insbl a1, 7, a2 # U : ch00000000000000 + + insbl t3, 6, a3 # U : 00ch000000000000 + or t5, t3, a1 # E : 000000000000chch + andnot a0, 7, v0 # E : align our loop pointer + lda t4, -1 # E : build garbage mask + + mskqh t4, a0, t4 # U : only want relevant part of first quad + or a2, a3, a2 # E : chch000000000000 + inswl a1, 2, t5 # E : 00000000chch0000 + inswl a1, 4, a3 # E : 0000chch00000000 + + or a1, a2, a1 # E : chch00000000chch + or a3, t5, t5 # E : 0000chchchch0000 + cmpbge zero, t0, t2 # E : bits set iff byte == zero + cmpbge zero, t4, t4 # E : bits set iff byte is garbage + + /* This quad is _very_ serialized. Lots of stalling happens */ + or t5, a1, a1 # E : chchchchchchchch + xor t0, a1, t1 # E : make bytes == c zero + cmpbge zero, t1, t3 # E : bits set iff byte == c + or t2, t3, t0 # E : bits set iff char match or zero match + + andnot t0, t4, t0 # E : clear garbage bits + cttz t3, a2 # U0 : speculative (in case we get a match) + nop # E : + bne t0, $found # U : Stall on t0 + + /* + * Yuk. This loop is going to stall like crazy waiting for the + * data to be loaded. Not much can be done about it unless it's + * unrolled multiple times - is that safe to do in kernel space? + * Or would exception handling recovery code do the trick here? + */ +$loop: ldq t0, 8(v0) # L : Latency=3 + addq v0, 8, v0 # E : + xor t0, a1, t1 # E : + cmpbge zero, t0, t2 # E : bits set iff byte == 0 + + cmpbge zero, t1, t3 # E : bits set iff byte == c + or t2, t3, t0 # E : + cttz t3, a2 # U0 : speculative (in case we get a match) + beq t0, $loop # U : + +$found: negq t0, t1 # E : clear all but least set bit + and t0, t1, t0 # E : + and t0, t3, t1 # E : bit set iff byte was the char + addq v0, a2, v0 # E : Add in the bit number from above + + cmoveq t1, $31, v0 # E : Two mapping slots, latency = 2 + nop + nop + ret # L0 : + + .end strchr + +#else .set noreorder .set noat @@ -68,3 +152,5 @@ ret # .. e1 : .end strchr + +#endif diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/strcpy.S linux-2.4.0-test9-perfpatch/arch/alpha/lib/strcpy.S --- linux-2.4.0-test9.ref/arch/alpha/lib/strcpy.S Fri Oct 18 04:53:20 1996 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/strcpy.S Mon Oct 9 10:52:54 2000 @@ -8,7 +8,7 @@ .text - .align 3 + .align 4 .globl strcpy .ent strcpy strcpy: @@ -18,5 +18,6 @@ mov $16, $0 # set up return value mov $26, $23 # set up return address br __stxcpy # do the copy + nop .end strcpy diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/strlen.S linux-2.4.0-test9-perfpatch/arch/alpha/lib/strlen.S --- linux-2.4.0-test9.ref/arch/alpha/lib/strlen.S Thu Jul 20 00:58:34 1995 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/strlen.S Thu Nov 9 16:36:19 2000 @@ -1,5 +1,6 @@ /* * strlen.S (c) 1995 David Mosberger (davidm@cs.arizona.edu) + * 21264 tuning by Rick Gorton * * Finds length of a 0-terminated string. Optimized for the * Alpha architecture: @@ -15,11 +16,51 @@ .set noreorder .set noat - .align 3 +#if (defined(__alpha_fix__) || defined(__alpha_cix__) && defined(CONFIG_ALPHA_EV67)) + +/* + * Much of the information about 21264 scheduling/coding comes from: + * Compiler Writer's Guide for the Alpha 21264 + * abbreviated as 'CWG' in other comments here + * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html + * Scheduling notation: + * E - either cluster + * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 + * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 + */ + + .globl strlen + .ent strlen + .align 4 +strlen: + ldq_u $1, 0($16) # L : load first quadword ($16 may be misaligned) + lda $2, -1($31) # E : + insqh $2, $16, $2 # U : + andnot $16, 7, $0 # E : + + or $2, $1, $1 # E : + cmpbge $31, $1, $2 # E : $2 <- bitmask: bit i == 1 <==> i-th byte == 0 + nop # E : + bne $2, $found # U : + +$loop: ldq $1, 8($0) # L : + addq $0, 8, $0 # E : addr += 8 + cmpbge $31, $1, $2 # E : + beq $2, $loop # U : + +$found: + cttz $2, $3 # U0 : + addq $0, $3, $0 # E : + subq $0, $16, $0 # E : + ret $31, ($26) # L0 : + + .end strlen +#else .globl strlen .ent strlen + .align 3 strlen: ldq_u $1, 0($16) # load first quadword ($16 may be misaligned) lda $2, -1($31) @@ -55,3 +96,4 @@ ret $31, ($26) .end strlen +#endif diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/strlen_user.S linux-2.4.0-test9-perfpatch/arch/alpha/lib/strlen_user.S --- linux-2.4.0-test9.ref/arch/alpha/lib/strlen_user.S Mon Oct 11 13:06:34 1999 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/strlen_user.S Thu Nov 9 16:36:19 2000 @@ -32,6 +32,85 @@ .ent __strlen_user .frame sp, 0, ra +#if (defined(__alpha_fix__) || defined(__alpha_cix__)) && defined(CONFIG_ALPHA_EV67) + +/* + * Much of the information about 21264 scheduling/coding comes from: + * Compiler Writer's Guide for the Alpha 21264 + * abbreviated as 'CWG' in other comments here + * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html + * Scheduling notation: + * E - either cluster + * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 + * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 + * Try not to change the actual algorithm if possible for consistency. + */ + + .align 4 +__strlen_user: + ldah a1, 32767(zero) # do not use plain strlen_user() for strings + # that might be almost 2 GB long; you should + # be using strnlen_user() instead + nop + nop + nop + + .globl __strnlen_user + + .align 4 +__strnlen_user: + ldgp $29,0($27) # E E : we do exceptions -- we need the gp. + /* Decomposes into lda/ldah */ + .prologue 1 + EX( ldq_u t0, 0(a0) ) # L : load first quadword (a0 may be misaligned) + lda t1, -1(zero) # E : + + insqh t1, a0, t1 # U : + andnot a0, 7, v0 # E : + or t1, t0, t0 # E : + subq a0, 1, a0 # E : get our +1 for the return + + cmpbge zero, t0, t1 # E : t1 <- bitmask: bit i == 1 <==> i-th byte == 0 + subq a1, 7, t2 # E : + subq a0, v0, t0 # E : + bne t1, $found # U : + + addq t2, t0, t2 # E : + addq a1, 1, a1 # E : + nop # E : + nop # E : + + .align 4 +$loop: ble t2, $limit # U : + EX( ldq t0, 8(v0) ) # L : + nop # E : + nop # E : + + cmpbge zero, t0, t1 # E : + subq t2, 8, t2 # E : + addq v0, 8, v0 # E : addr += 8 + beq t1, $loop # U : + +$found: cttz t1, t2 # U0 : + addq v0, t2, v0 # E : + subq v0, a0, v0 # E : + ret # L0 : + +$exception: + nop + nop + nop + ret + + .align 4 # currently redundant +$limit: + nop + nop + subq a1, t2, v0 + ret + + .end __strlen_user +#else .align 3 __strlen_user: ldah a1, 32767(zero) # do not use plain strlen_user() for strings @@ -90,3 +169,5 @@ ret .end __strlen_user + +#endif diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/strncat.S linux-2.4.0-test9-perfpatch/arch/alpha/lib/strncat.S --- linux-2.4.0-test9.ref/arch/alpha/lib/strncat.S Thu Feb 6 07:42:35 1997 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/strncat.S Thu Nov 9 16:36:19 2000 @@ -12,6 +12,90 @@ .text +#if (defined(__alpha_fix__) || defined(__alpha_cix__)) && defined(CONFIG_ALPHA_EV67) + +/* + * Much of the information about 21264 scheduling/coding comes from: + * Compiler Writer's Guide for the Alpha 21264 + * abbreviated as 'CWG' in other comments here + * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html + * Scheduling notation: + * E - either cluster + * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 + * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 + * Try not to change the actual algorithm if possible for consistency. + */ + + .align 3 + .globl strncat + .ent strncat +strncat: + .frame $30, 0, $26 + .prologue 0 + + mov $16, $0 # set up return value + beq $18, $zerocount # U : + /* Find the end of the string. */ + ldq_u $1, 0($16) # L : load first quadword ($16 may be misaligned) + lda $2, -1($31) # E : + + insqh $2, $0, $2 # U : + andnot $16, 7, $16 # E : + nop # E : + or $2, $1, $1 # E : + + nop # E : + nop # E : + cmpbge $31, $1, $2 # E : bits set iff byte == 0 + bne $2, $found # U : + +$loop: ldq $1, 8($16) # L : + addq $16, 8, $16 # E : + cmpbge $31, $1, $2 # E : + beq $2, $loop # U : + +$found: cttz $2, $3 # U0 : + addq $16, $3, $16 # E : + nop # E : + bsr $23, __stxncpy # L0 :/* Now do the append. */ + + /* Worry about the null termination. */ + + zapnot $1, $27, $2 # U : was last byte a null? + cmplt $27, $24, $5 # E : did we fill the buffer completely? + bne $2, 0f # U : + ret # L0 : + +0: or $5, $18, $2 # E : + nop + bne $2, 2f # U : + and $24, 0x80, $3 # E : no zero next byte + + nop # E : + bne $3, 1f # U : + /* Here there are bytes left in the current word. Clear one. */ + addq $24, $24, $24 # E : end-of-count bit <<= 1 + nop # E : + +2: zap $1, $24, $1 # U : + nop # E : + stq_u $1, 0($16) # L : + ret # L0 : + +1: /* Here we must clear the first byte of the next DST word */ + stb $31, 8($16) # L : + nop # E : + nop # E : + ret # L0 : + +$zerocount: + nop # E : + nop # E : + nop # E : + ret # L0 : + + .end strncat +#else .align 3 .globl strncat .ent strncat @@ -82,3 +166,4 @@ ret .end strncat +#endif diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/strncpy.S linux-2.4.0-test9-perfpatch/arch/alpha/lib/strncpy.S --- linux-2.4.0-test9.ref/arch/alpha/lib/strncpy.S Fri Oct 18 04:53:20 1996 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/strncpy.S Mon Oct 9 10:52:55 2000 @@ -16,7 +16,7 @@ .text - .align 3 + .align 4 .globl strncpy .ent strncpy strncpy: @@ -27,6 +27,10 @@ beq $18, 0f mov $26, $23 # set return address br __stxncpy # do the work of the copy + 0: ret + nop + nop + nop .end strncpy diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/strncpy_from_user.S linux-2.4.0-test9-perfpatch/arch/alpha/lib/strncpy_from_user.S --- linux-2.4.0-test9.ref/arch/alpha/lib/strncpy_from_user.S Sat Apr 24 20:54:08 1999 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/strncpy_from_user.S Mon Oct 9 10:52:55 2000 @@ -1,6 +1,7 @@ /* * arch/alpha/lib/strncpy_from_user.S * Contributed by Richard Henderson (rth@tamu.edu) + * 21264 version by Rick Gorton * * Just like strncpy except in the return value: * @@ -28,6 +29,7 @@ .set noreorder .text +#ifndef __alpha_cix__ .globl __strncpy_from_user .ent __strncpy_from_user .frame $30, 0, $26 @@ -339,3 +341,405 @@ ret .end __strncpy_from_user + +#else + +/* + * Much of the information about 21264 scheduling/coding comes from: + * Compiler Writer's Guide for the Alpha 21264 + * abbreviated as 'CWG' in other comments here + * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html + * Scheduling notation: + * E - either cluster + * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 + * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 + * A bunch of instructions got moved and temp registers were changed + * to aid in scheduling. Control flow was also re-arranged to eliminate + * branches, and to provide longer code sequences to enable better scheduling. + * A total rewrite (using byte load/stores for start & tail sequences) + * is desirable, but very difficult to do without a from-scratch rewrite. + * Save that for the future. + */ + + + .globl __strncpy_from_user + .ent __strncpy_from_user + .frame $30, 0, $26 + .prologue 1 + + .align 4 +__strncpy_from_user: + ldgp $29, 0($27) # E E : becomes 2 instructions (for exceptions) + and a0, 7, t3 # E : find dest misalignment + beq a2, $zerolength # U : + + /* Are source and destination co-aligned? */ + mov a0, v0 # E : save the string start + xor a0, a1, t4 # E : + EX( ldq_u t1, 0(a1) ) # L : Latency=3 load first quadword + ldq_u t0, 0(a0) # L : load first (partial) aligned dest quadword + + addq a2, t3, a2 # E : bias count by dest misalignment + subq a2, 1, a3 # E : + addq zero, 1, t10 # E : + and t4, 7, t4 # E : misalignment between the two + + and a3, 7, t6 # E : number of tail bytes + sll t10, t6, t10 # E : t10 = bitmask of last count byte + bne t4, $unaligned # U : + lda t2, -1 # E : build a mask against false zero + + /* + * We are co-aligned; take care of a partial first word. + * On entry to this basic block: + * t0 == the first destination word for masking back in + * t1 == the first source word. + */ + + srl a3, 3, a2 # E : a2 = loop counter = (count - 1)/8 + addq a1, 8, a1 # E : + mskqh t2, a1, t2 # U : detection in the src word + nop + + /* Create the 1st output word and detect 0's in the 1st input word. */ + mskqh t1, a1, t3 # U : + mskql t0, a1, t0 # U : assemble the first output word + ornot t1, t2, t2 # E : + nop + + cmpbge zero, t2, t8 # E : bits set iff null found + or t0, t3, t0 # E : + beq a2, $a_eoc # U : + bne t8, $a_eos # U : 2nd branch in a quad. Bad. + + /* On entry to this basic block: + * t0 == a source quad not containing a null. + * a0 - current aligned destination address + * a1 - current aligned source address + * a2 - count of quadwords to move. + * NOTE: Loop improvement - unrolling this is going to be + * a huge win, since we're going to stall otherwise. + * Fix this later. For _really_ large copies, look + * at using wh64 on a look-ahead basis. See the code + * in clear_user.S and copy_user.S. + * Presumably, since (a0) and (a1) do not overlap (by C definition) + * Lots of nops here: + * - Separate loads from stores + * - Keep it to 1 branch/quadpack so the branch predictor + * can train. + */ +$a_loop: + stq_u t0, 0(a0) # L : + addq a0, 8, a0 # E : + nop + subq a2, 1, a2 # E : + + EX( ldq_u t0, 0(a1) ) # L : + addq a1, 8, a1 # E : + cmpbge zero, t0, t8 # E : Stall 2 cycles on t0 + beq a2, $a_eoc # U : + + beq t8, $a_loop # U : + nop + nop + nop + + /* Take care of the final (partial) word store. At this point + * the end-of-count bit is set in t8 iff it applies. + * + * On entry to this basic block we have: + * t0 == the source word containing the null + * t8 == the cmpbge mask that found it. + */ +$a_eos: + negq t8, t12 # E : find low bit set + and t8, t12, t12 # E : + + /* We're doing a partial word store and so need to combine + our source and original destination words. */ + ldq_u t1, 0(a0) # L : + subq t12, 1, t6 # E : + + or t12, t6, t8 # E : + zapnot t0, t8, t0 # U : clear src bytes > null + zap t1, t8, t1 # U : clear dst bytes <= null + or t0, t1, t0 # E : + + stq_u t0, 0(a0) # L : + br $finish_up # L0 : + nop + nop + + /* Add the end-of-count bit to the eos detection bitmask. */ + .align 4 +$a_eoc: + or t10, t8, t8 + br $a_eos + nop + nop + + +/* The source and destination are not co-aligned. Align the destination + and cope. We have to be very careful about not reading too much and + causing a SEGV. */ + + .align 4 +$u_head: + /* We know just enough now to be able to assemble the first + full source word. We can still find a zero at the end of it + that prevents us from outputting the whole thing. + + On entry to this basic block: + t0 == the first dest word, unmasked + t1 == the shifted low bits of the first source word + t6 == bytemask that is -1 in dest word bytes */ + + EX( ldq_u t2, 8(a1) ) # L : load second src word + addq a1, 8, a1 # E : + mskql t0, a0, t0 # U : mask trailing garbage in dst + extqh t2, a1, t4 # U : + + or t1, t4, t1 # E : first aligned src word complete + mskqh t1, a0, t1 # U : mask leading garbage in src + or t0, t1, t0 # E : first output word complete + or t0, t6, t6 # E : mask original data for zero test + + cmpbge zero, t6, t8 # E : + beq a2, $u_eocfin # U : + bne t8, $u_final # U : bad news - 2nd branch in a quad + lda t6, -1 # E : mask out the bits we have + + mskql t6, a1, t6 # U : already seen + stq_u t0, 0(a0) # L : store first output word + or t6, t2, t2 # E : + cmpbge zero, t2, t8 # E : find nulls in second partial + + addq a0, 8, a0 # E : + subq a2, 1, a2 # E : + bne t8, $u_late_head_exit # U : + nop + + /* Finally, we've got all the stupid leading edge cases taken care + of and we can set up to enter the main loop. */ + + extql t2, a1, t1 # U : position hi-bits of lo word + EX( ldq_u t2, 8(a1) ) # L : read next high-order source word + addq a1, 8, a1 # E : + cmpbge zero, t2, t8 # E : + + beq a2, $u_eoc # U : + bne t8, $u_eos # U : + nop + nop + + /* Unaligned copy main loop. In order to avoid reading too much, + the loop is structured to detect zeros in aligned source words. + This has, unfortunately, effectively pulled half of a loop + iteration out into the head and half into the tail, but it does + prevent nastiness from accumulating in the very thing we want + to run as fast as possible. + + On entry to this basic block: + t1 == the shifted high-order bits from the previous source word + t2 == the unshifted current source word + + We further know that t2 does not contain a null terminator. */ + + /* + * Extra nops here: + * separate load quads from store quads + * only one branch/quad to permit predictor training + */ + + .align 4 +$u_loop: + extqh t2, a1, t0 # U : extract high bits for current word + addq a1, 8, a1 # E : + extql t2, a1, t3 # U : extract low bits for next time + addq a0, 8, a0 # E : + + or t0, t1, t0 # E : current dst word now complete + EX( ldq_u t2, 0(a1) ) # L : load high word for next time + subq a2, 1, a2 # E : + nop + + stq_u t0, -8(a0) # L : save the current word + mov t3, t1 # E : + cmpbge zero, t2, t8 # E : test new word for eos + beq a2, $u_eoc # U : + + beq t8, $u_loop # U : + nop + nop + nop + + /* We've found a zero somewhere in the source word we just read. + If it resides in the lower half, we have one (probably partial) + word to write out, and if it resides in the upper half, we + have one full and one partial word left to write out. + + On entry to this basic block: + t1 == the shifted high-order bits from the previous source word + t2 == the unshifted current source word. */ + .align 4 +$u_eos: + extqh t2, a1, t0 # U : + or t0, t1, t0 # E : first (partial) source word complete + cmpbge zero, t0, t8 # E : is the null in this first bit? + nop + + bne t8, $u_final # U : + stq_u t0, 0(a0) # L : the null was in the high-order bits + addq a0, 8, a0 # E : + subq a2, 1, a2 # E : + + .align 4 +$u_late_head_exit: + extql t2, a1, t0 # U : + cmpbge zero, t0, t8 # E : + or t8, t10, t6 # E : + cmoveq a2, t6, t8 # E : + + /* Take care of a final (probably partial) result word. + On entry to this basic block: + t0 == assembled source word + t8 == cmpbge mask that found the null. */ + .align 4 +$u_final: + negq t8, t6 # E : isolate low bit set + and t6, t8, t12 # E : + ldq_u t1, 0(a0) # L : + subq t12, 1, t6 # E : + + or t6, t12, t8 # E : + zapnot t0, t8, t0 # U : kill source bytes > null + zap t1, t8, t1 # U : kill dest bytes <= null + or t0, t1, t0 # E : + + stq_u t0, 0(a0) # E : + br $finish_up # U : + nop + nop + + .align 4 +$u_eoc: # end-of-count + extqh t2, a1, t0 # U : + or t0, t1, t0 # E : + cmpbge zero, t0, t8 # E : + nop + + .align 4 +$u_eocfin: # end-of-count, final word + or t10, t8, t8 # E : + br $u_final # U : + nop + nop + + /* Unaligned copy entry point. */ + .align 4 +$unaligned: + + srl a3, 3, a2 # U : a2 = loop counter = (count - 1)/8 + and a0, 7, t4 # E : find dest misalignment + and a1, 7, t5 # E : find src misalignment + mov zero, t0 # E : + + /* Conditionally load the first destination word and a bytemask + with 0xff indicating that the destination byte is sacrosanct. */ + + mov zero, t6 # E : + beq t4, 1f # U : + ldq_u t0, 0(a0) # L : + lda t6, -1 # E : + + mskql t6, a0, t6 # E : + nop + nop + nop + + .align 4 +1: + subq a1, t4, a1 # E : sub dest misalignment from src addr + /* If source misalignment is larger than dest misalignment, we need + extra startup checks to avoid SEGV. */ + cmplt t4, t5, t12 # E : + extql t1, a1, t1 # U : shift src into place + lda t2, -1 # E : for creating masks later + + beq t12, $u_head # U : + mskqh t2, t5, t2 # U : begin src byte validity mask + cmpbge zero, t1, t8 # E : is there a zero? + nop + + extql t2, a1, t2 # U : + or t8, t10, t5 # E : test for end-of-count too + cmpbge zero, t2, t3 # E : + cmoveq a2, t5, t8 # E : Latency=2, extra map slot + + nop # E : goes with cmov + andnot t8, t3, t8 # E : + beq t8, $u_head # U : + nop + + /* At this point we've found a zero in the first partial word of + the source. We need to isolate the valid source data and mask + it into the original destination data. (Incidentally, we know + that we'll need at least one byte of that original dest word.) */ + + ldq_u t0, 0(a0) # L : + negq t8, t6 # E : build bitmask of bytes <= zero + mskqh t1, t4, t1 # U : + and t6, t8, t12 # E : + + subq t12, 1, t6 # E : + or t6, t12, t8 # E : + zapnot t2, t8, t2 # U : prepare source word; mirror changes + zapnot t1, t8, t1 # U : to source validity mask + + andnot t0, t2, t0 # E : zero place for source to reside + or t0, t1, t0 # E : and put it there + stq_u t0, 0(a0) # L : + nop + + .align 4 +$finish_up: + zapnot t0, t12, t4 # U : was last byte written null? + and t12, 0xf0, t3 # E : binary search for the address of the + cmovne t4, 1, t4 # E : Latency=2, extra map slot + nop # E : with cmovne + + and t12, 0xcc, t2 # E : last byte written + and t12, 0xaa, t1 # E : + cmovne t3, 4, t3 # E : Latency=2, extra map slot + nop # E : with cmovne + + bic a0, 7, t0 + cmovne t2, 2, t2 # E : Latency=2, extra map slot + nop # E : with cmovne + nop + + cmovne t1, 1, t1 # E : Latency=2, extra map slot + nop # E : with cmovne + addq t0, t3, t0 # E : + addq t1, t2, t1 # E : + + addq t0, t1, t0 # E : + addq t0, t4, t0 # add one if we filled the buffer + subq t0, v0, v0 # find string length + ret # L0 : + + .align 4 +$zerolength: + nop + nop + nop + clr v0 + +$exception: + nop + nop + nop + ret + + .end __strncpy_from_user +#endif diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/strrchr.S linux-2.4.0-test9-perfpatch/arch/alpha/lib/strrchr.S --- linux-2.4.0-test9.ref/arch/alpha/lib/strrchr.S Wed Nov 19 17:07:33 1997 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/strrchr.S Thu Nov 9 16:36:19 2000 @@ -1,6 +1,7 @@ /* * arch/alpha/lib/strrchr.S * Contributed by Richard Henderson (rth@tamu.edu) + * 21264 version by Rick Gorton * * Return the address of the last occurrance of a given character * within a null-terminated string, or null if it is not found. @@ -11,6 +12,111 @@ .set noreorder .set noat +#if (defined(__alpha_fix__) || defined(__alpha_cix__)) && defined(CONFIG_ALPHA_EV67) + +/* + * Much of the information about 21264 scheduling/coding comes from: + * Compiler Writer's Guide for the Alpha 21264 + * abbreviated as 'CWG' in other comments here + * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html + * Scheduling notation: + * E - either cluster + * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 + * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 + * In this case, the algorithm has been significanly rewritten + */ + + .align 4 + .ent strrchr + .globl strrchr +strrchr: + .frame sp, 0, ra + .prologue 0 + + and a1, 0xff, t2 # E : 00000000000000ch + insbl a1, 1, t4 # U : 000000000000ch00 + insbl a1, 2, t5 # U : 0000000000ch0000 + ldq_u t0, 0(a0) # L : load first quadword Latency=3 + + mov zero, v0 # E : Initially null: last address of match + or t2, t4, a1 # E : 000000000000chch + sll t5, 8, t3 # U : 00000000ch000000 + lda t8, -1 # E : Start to build garbage mask + + or t5, t3, t3 # E : 00000000chch0000 + sll a1, 16, t2 # U : 0000chch00000000 + sll a1, 24, t4 # U : chch000000000000 + mskqh t8, a0, t8 # U : Complete garbage mask + + or t4, a1, a1 # E : chch00000000chch + or t2, t3, t2 # E : 0000chchchch0000 + cmpbge zero, t0, t1 # E : bits set iff byte == zero + andnot a0, 7, a0 # E : align source addr + + cmpbge zero, t8, t8 # E : bits set iff byte is garbage + or a1, t2, a1 # E : chchchchchchchch + xor t0, a1, t2 # E : make bytes == c zero + nop + + ldq t0, 8(a0) # L : load first full quadword Latency=3 + cmpbge zero, t2, t3 # E : bits set iff byte == c + andnot t1, t8, t1 # E : clear garbage from null test + nop + + cttz t1, t4 # U0 : Speculative - Latency=3 + /* byte number of the match (or 0x40) */ + andnot t3, t8, t3 # E : clear garbage from char test + nop + beq t1, $loop # U : More to go? + + cttz t3, t5 # U0 : Latency=3 + /* byte number of the terminator (seen if we got here) */ + nop + nop + nop + + /* 2 cycle stall for t5, 1 cycle stall for t4 */ + addq a0, t5, a2 # E : Speculative - address if a match did happen + cmple t5, t4, t6 # E : &(match) <= & (NULL), set t6 + nop + nop + + cmovne t6, a2, v0 # E : When seen, new 'last'. Latency 2, extra map slot + nop # E : keep with cmovne + nop + ret # L0 : + + /* Character search main loop */ +$loop: + ldq a3, 16(a0) # L : load next quadword Latency=3 + /* 2 cycle stall for initial t0, no stall in the loop */ + nop + xor t0, a1, t2 # E : + cmpbge zero, t0, t1 # E : bits set iff byte == zero + + or t1, t1, t7 # E : to avoid cross-cluster stalls + cmpbge zero, t2, t3 # E : bits set iff byte == c + cttz t1, t4 # U0 : Latency=3 byte number of the match (or 0x40) + nop + + addq a0, 8, a0 # E : Point at next quad + cttz t3, t5 # U0 : Latency=3 byte number of the null (or 0x40) + nop + nop + + addq a0, t5, a2 # E : Speculative - address if a match did happen + nop + or a3, a3, t0 # E : Copy of next data into current register + bne t7, $loop # U : go until we find a zero + + /* 2 cycle stall for t5 */ + cmple t5, t4, t6 # E : &(match) <= & (NULL), set t6 + cmovne t6, a2, v0 # E : When seen, new 'last'. Latency 2, extra map slot + nop # E : keep with cmovne + ret # L0 : + + .end strrchr +#else .align 3 .ent strrchr .globl strrchr @@ -85,3 +191,4 @@ ret # .. e1 : .end strrchr +#endif diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/stxcpy.S linux-2.4.0-test9-perfpatch/arch/alpha/lib/stxcpy.S --- linux-2.4.0-test9.ref/arch/alpha/lib/stxcpy.S Thu Feb 6 07:42:35 1997 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/stxcpy.S Mon Oct 9 10:52:55 2000 @@ -31,6 +31,8 @@ middle of the procedure descriptor. Work around this by putting the aligned copy in its own procedure descriptor */ +#ifndef __alpha_ev6__ + .ent stxcpy_aligned .align 3 stxcpy_aligned: @@ -286,3 +288,296 @@ ret (t9) # e1 : .end __stxcpy + +#else + +/* + * Much of the information about 21264 scheduling/coding comes from: + * Compiler Writer's Guide for the Alpha 21264 + * abbreviated as 'CWG' in other comments here + * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html + * Scheduling notation: + * E - either cluster + * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 + * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 + * Try not to change the actual algorithm if possible for consistency. + */ + + .ent stxcpy_aligned + .align 4 +stxcpy_aligned: + .frame sp, 0, t9 + .prologue 0 + + /* On entry to this basic block: + t0 == the first destination word for masking back in + t1 == the first source word. */ + + /* Create the 1st output word and detect 0's in the 1st input word. */ + lda t2, -1 # E : build a mask against false zero + mskqh t2, a1, t2 # U : detection in the src word (stall) + mskqh t1, a1, t3 # U : + ornot t1, t2, t2 # E : (stall) + + mskql t0, a1, t0 # U : assemble the first output word + cmpbge zero, t2, t8 # E : bits set iff null found + or t0, t3, t1 # E : (stall) + bne t8, $a_eos # U : (stall) + + /* On entry to this basic block: + t0 == the first destination word for masking back in + t1 == a source word not containing a null. */ + /* Nops here to separate store quads from load quads */ + +$a_loop: + stq_u t1, 0(a0) # L : + addq a0, 8, a0 # E : + nop + nop + + ldq_u t1, 0(a1) # L : Latency=3 + addq a1, 8, a1 # E : + cmpbge zero, t1, t8 # E : (3 cycle stall) + beq t8, $a_loop # U : (stall for t8) + + /* Take care of the final (partial) word store. + On entry to this basic block we have: + t1 == the source word containing the null + t8 == the cmpbge mask that found it. */ +$a_eos: + negq t8, t6 # E : find low bit set + and t8, t6, t12 # E : (stall) + /* For the sake of the cache, don't read a destination word + if we're not going to need it. */ + and t12, 0x80, t6 # E : (stall) + bne t6, 1f # U : (stall) + + /* We're doing a partial word store and so need to combine + our source and original destination words. */ + ldq_u t0, 0(a0) # L : Latency=3 + subq t12, 1, t6 # E : + zapnot t1, t6, t1 # U : clear src bytes >= null (stall) + or t12, t6, t8 # E : (stall) + + zap t0, t8, t0 # E : clear dst bytes <= null + or t0, t1, t1 # E : (stall) + nop + nop + +1: stq_u t1, 0(a0) # L : + ret (t9) # L0 : Latency=3 + nop + nop + + .end stxcpy_aligned + + .align 4 + .ent __stxcpy + .globl __stxcpy +__stxcpy: + .frame sp, 0, t9 + .prologue 0 + + /* Are source and destination co-aligned? */ + xor a0, a1, t0 # E : + unop # E : + and t0, 7, t0 # E : (stall) + bne t0, $unaligned # U : (stall) + + /* We are co-aligned; take care of a partial first word. */ + ldq_u t1, 0(a1) # L : load first src word + and a0, 7, t0 # E : take care not to load a word ... + addq a1, 8, a1 # E : + beq t0, stxcpy_aligned # U : ... if we wont need it (stall) + + ldq_u t0, 0(a0) # L : + br stxcpy_aligned # L0 : Latency=3 + nop + nop + + +/* The source and destination are not co-aligned. Align the destination + and cope. We have to be very careful about not reading too much and + causing a SEGV. */ + + .align 4 +$u_head: + /* We know just enough now to be able to assemble the first + full source word. We can still find a zero at the end of it + that prevents us from outputting the whole thing. + + On entry to this basic block: + t0 == the first dest word, for masking back in, if needed else 0 + t1 == the low bits of the first source word + t6 == bytemask that is -1 in dest word bytes */ + + ldq_u t2, 8(a1) # L : + addq a1, 8, a1 # E : + extql t1, a1, t1 # U : (stall on a1) + extqh t2, a1, t4 # U : (stall on a1) + + mskql t0, a0, t0 # U : + or t1, t4, t1 # E : + mskqh t1, a0, t1 # U : (stall on t1) + or t0, t1, t1 # E : (stall on t1) + + or t1, t6, t6 # E : + cmpbge zero, t6, t8 # E : (stall) + lda t6, -1 # E : for masking just below + bne t8, $u_final # U : (stall) + + mskql t6, a1, t6 # U : mask out the bits we have + or t6, t2, t2 # E : already extracted before (stall) + cmpbge zero, t2, t8 # E : testing eos (stall) + bne t8, $u_late_head_exit # U : (stall) + + /* Finally, we've got all the stupid leading edge cases taken care + of and we can set up to enter the main loop. */ + + stq_u t1, 0(a0) # L : store first output word + addq a0, 8, a0 # E : + extql t2, a1, t0 # U : position ho-bits of lo word + ldq_u t2, 8(a1) # U : read next high-order source word + + addq a1, 8, a1 # E : + cmpbge zero, t2, t8 # E : (stall for t2) + nop # E : + bne t8, $u_eos # U : (stall) + + /* Unaligned copy main loop. In order to avoid reading too much, + the loop is structured to detect zeros in aligned source words. + This has, unfortunately, effectively pulled half of a loop + iteration out into the head and half into the tail, but it does + prevent nastiness from accumulating in the very thing we want + to run as fast as possible. + + On entry to this basic block: + t0 == the shifted high-order bits from the previous source word + t2 == the unshifted current source word + + We further know that t2 does not contain a null terminator. */ + + .align 3 +$u_loop: + extqh t2, a1, t1 # U : extract high bits for current word + addq a1, 8, a1 # E : (stall) + extql t2, a1, t3 # U : extract low bits for next time (stall) + addq a0, 8, a0 # E : + + or t0, t1, t1 # E : current dst word now complete + ldq_u t2, 0(a1) # L : Latency=3 load high word for next time + stq_u t1, -8(a0) # L : save the current word (stall) + mov t3, t0 # E : + + cmpbge zero, t2, t8 # E : test new word for eos + beq t8, $u_loop # U : (stall) + nop + nop + + /* We've found a zero somewhere in the source word we just read. + If it resides in the lower half, we have one (probably partial) + word to write out, and if it resides in the upper half, we + have one full and one partial word left to write out. + + On entry to this basic block: + t0 == the shifted high-order bits from the previous source word + t2 == the unshifted current source word. */ +$u_eos: + extqh t2, a1, t1 # U : + or t0, t1, t1 # E : first (partial) source word complete (stall) + cmpbge zero, t1, t8 # E : is the null in this first bit? (stall) + bne t8, $u_final # U : (stall) + +$u_late_head_exit: + stq_u t1, 0(a0) # L : the null was in the high-order bits + addq a0, 8, a0 # E : + extql t2, a1, t1 # U : + cmpbge zero, t1, t8 # E : (stall) + + /* Take care of a final (probably partial) result word. + On entry to this basic block: + t1 == assembled source word + t8 == cmpbge mask that found the null. */ +$u_final: + negq t8, t6 # E : isolate low bit set + and t6, t8, t12 # E : (stall) + and t12, 0x80, t6 # E : avoid dest word load if we can (stall) + bne t6, 1f # U : (stall) + + ldq_u t0, 0(a0) # E : + subq t12, 1, t6 # E : + or t6, t12, t8 # E : (stall) + zapnot t1, t6, t1 # U : kill source bytes >= null (stall) + + zap t0, t8, t0 # U : kill dest bytes <= null (2 cycle data stall) + or t0, t1, t1 # E : (stall) + nop + nop + +1: stq_u t1, 0(a0) # L : + ret (t9) # L0 : Latency=3 + nop + nop + + /* Unaligned copy entry point. */ + .align 4 +$unaligned: + + ldq_u t1, 0(a1) # L : load first source word + and a0, 7, t4 # E : find dest misalignment + and a1, 7, t5 # E : find src misalignment + /* Conditionally load the first destination word and a bytemask + with 0xff indicating that the destination byte is sacrosanct. */ + mov zero, t0 # E : + + mov zero, t6 # E : + beq t4, 1f # U : + ldq_u t0, 0(a0) # L : + lda t6, -1 # E : + + mskql t6, a0, t6 # U : + nop + nop + nop +1: + subq a1, t4, a1 # E : sub dest misalignment from src addr + /* If source misalignment is larger than dest misalignment, we need + extra startup checks to avoid SEGV. */ + cmplt t4, t5, t12 # E : + beq t12, $u_head # U : + lda t2, -1 # E : mask out leading garbage in source + + mskqh t2, t5, t2 # U : + ornot t1, t2, t3 # E : (stall) + cmpbge zero, t3, t8 # E : is there a zero? (stall) + beq t8, $u_head # U : (stall) + + /* At this point we've found a zero in the first partial word of + the source. We need to isolate the valid source data and mask + it into the original destination data. (Incidentally, we know + that we'll need at least one byte of that original dest word.) */ + + ldq_u t0, 0(a0) # L : + negq t8, t6 # E : build bitmask of bytes <= zero + and t6, t8, t12 # E : (stall) + and a1, 7, t5 # E : + + subq t12, 1, t6 # E : + or t6, t12, t8 # E : (stall) + srl t12, t5, t12 # U : adjust final null return value + zapnot t2, t8, t2 # U : prepare source word; mirror changes (stall) + + and t1, t2, t1 # E : to source validity mask + extql t2, a1, t2 # U : + extql t1, a1, t1 # U : (stall) + andnot t0, t2, t0 # .. e1 : zero place for source to reside (stall) + + or t0, t1, t1 # e1 : and put it there + stq_u t1, 0(a0) # .. e0 : (stall) + ret (t9) # e1 : + nop + + .end __stxcpy + +#endif + diff -Naur linux-2.4.0-test9.ref/arch/alpha/lib/stxncpy.S linux-2.4.0-test9-perfpatch/arch/alpha/lib/stxncpy.S --- linux-2.4.0-test9.ref/arch/alpha/lib/stxncpy.S Thu Feb 6 07:51:09 1997 +++ linux-2.4.0-test9-perfpatch/arch/alpha/lib/stxncpy.S Mon Oct 9 10:52:55 2000 @@ -39,6 +39,7 @@ middle of the procedure descriptor. Work around this by putting the aligned copy in its own procedure descriptor */ +#ifndef __alpha_ev6__ .ent stxncpy_aligned .align 3 stxncpy_aligned: @@ -327,3 +328,365 @@ ret (t9) # .. e1 : .end __stxncpy + +#else + +/* + * Much of the information about 21264 scheduling/coding comes from: + * Compiler Writer's Guide for the Alpha 21264 + * abbreviated as 'CWG' in other comments here + * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html + * Scheduling notation: + * E - either cluster + * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 + * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 + * Try not to change the actual algorithm if possible for consistency. + */ + + .ent stxncpy_aligned + .align 4 +stxncpy_aligned: + .frame sp, 0, t9, 0 + .prologue 0 + + /* On entry to this basic block: + t0 == the first destination word for masking back in + t1 == the first source word. */ + + /* Create the 1st output word and detect 0's in the 1st input word. */ + lda t2, -1 # E : build a mask against false zero + mskqh t2, a1, t2 # U : detection in the src word (stall) + mskqh t1, a1, t3 # U : + ornot t1, t2, t2 # E : (stall) + + mskql t0, a1, t0 # U : assemble the first output word + cmpbge zero, t2, t8 # E : bits set iff null found + or t0, t3, t0 # E : (stall) + beq a2, $a_eoc # U : + + bne t8, $a_eos # U : + nop + nop + nop + + /* On entry to this basic block: + t0 == a source word not containing a null. */ + + /* + * nops here to: + * separate store quads from load quads + * limit of 1 bcond/quad to permit training + */ +$a_loop: + stq_u t0, 0(a0) # L : + addq a0, 8, a0 # E : + subq a2, 1, a2 # E : + nop + + ldq_u t0, 0(a1) # L : + addq a1, 8, a1 # E : + cmpbge zero, t0, t8 # E : + beq a2, $a_eoc # U : + + beq t8, $a_loop # U : + nop + nop + nop + + /* Take care of the final (partial) word store. At this point + the end-of-count bit is set in t8 iff it applies. + + On entry to this basic block we have: + t0 == the source word containing the null + t8 == the cmpbge mask that found it. */ + +$a_eos: + negq t8, t12 # E : find low bit set + and t8, t12, t12 # E : (stall) + /* For the sake of the cache, don't read a destination word + if we're not going to need it. */ + and t12, 0x80, t6 # E : (stall) + bne t6, 1f # U : (stall) + + /* We're doing a partial word store and so need to combine + our source and original destination words. */ + ldq_u t1, 0(a0) # L : + subq t12, 1, t6 # E : + or t12, t6, t8 # E : (stall) + zapnot t0, t8, t0 # U : clear src bytes > null (stall) + + zap t1, t8, t1 # .. e1 : clear dst bytes <= null + or t0, t1, t0 # e1 : (stall) + nop + nop + +1: stq_u t0, 0(a0) # L : + ret (t9) # L0 : Latency=3 + nop + nop + + /* Add the end-of-count bit to the eos detection bitmask. */ +$a_eoc: + or t10, t8, t8 # E : + br $a_eos # L0 : Latency=3 + nop + nop + + .end stxncpy_aligned + + .align 4 + .ent __stxncpy + .globl __stxncpy +__stxncpy: + .frame sp, 0, t9, 0 + .prologue 0 + + /* Are source and destination co-aligned? */ + xor a0, a1, t1 # E : + and a0, 7, t0 # E : find dest misalignment + and t1, 7, t1 # E : (stall) + addq a2, t0, a2 # E : bias count by dest misalignment (stall) + + subq a2, 1, a2 # E : + and a2, 7, t2 # E : (stall) + srl a2, 3, a2 # U : a2 = loop counter = (count - 1)/8 (stall) + addq zero, 1, t10 # E : + + sll t10, t2, t10 # U : t10 = bitmask of last count byte + bne t1, $unaligned # U : + /* We are co-aligned; take care of a partial first word. */ + ldq_u t1, 0(a1) # L : load first src word + addq a1, 8, a1 # E : + + beq t0, stxncpy_aligned # U : avoid loading dest word if not needed + ldq_u t0, 0(a0) # L : + nop + nop + + br stxncpy_aligned # .. e1 : + nop + nop + nop + + + +/* The source and destination are not co-aligned. Align the destination + and cope. We have to be very careful about not reading too much and + causing a SEGV. */ + + .align 4 +$u_head: + /* We know just enough now to be able to assemble the first + full source word. We can still find a zero at the end of it + that prevents us from outputting the whole thing. + + On entry to this basic block: + t0 == the first dest word, unmasked + t1 == the shifted low bits of the first source word + t6 == bytemask that is -1 in dest word bytes */ + + ldq_u t2, 8(a1) # L : Latency=3 load second src word + addq a1, 8, a1 # E : + mskql t0, a0, t0 # U : mask trailing garbage in dst + extqh t2, a1, t4 # U : (3 cycle stall on t2) + + or t1, t4, t1 # E : first aligned src word complete (stall) + mskqh t1, a0, t1 # U : mask leading garbage in src (stall) + or t0, t1, t0 # E : first output word complete (stall) + or t0, t6, t6 # E : mask original data for zero test (stall) + + cmpbge zero, t6, t8 # E : + beq a2, $u_eocfin # U : + nop + nop + + bne t8, $u_final # U : + lda t6, -1 # E : mask out the bits we have + mskql t6, a1, t6 # U : already seen (stall) + stq_u t0, 0(a0) # L : store first output word + + or t6, t2, t2 # E : + cmpbge zero, t2, t8 # E : find nulls in second partial (stall) + addq a0, 8, a0 # E : + subq a2, 1, a2 # E : + + bne t8, $u_late_head_exit # U : + /* Finally, we've got all the stupid leading edge cases taken care + of and we can set up to enter the main loop. */ + extql t2, a1, t1 # U : position hi-bits of lo word + ldq_u t2, 8(a1) # L : read next high-order source word + addq a1, 8, a1 # E : + + cmpbge zero, t2, t8 # E : (stall) + beq a2, $u_eoc # U : + nop + nop + + bne t8, $u_eos # e1 : + nop + nop + nop + + /* Unaligned copy main loop. In order to avoid reading too much, + the loop is structured to detect zeros in aligned source words. + This has, unfortunately, effectively pulled half of a loop + iteration out into the head and half into the tail, but it does + prevent nastiness from accumulating in the very thing we want + to run as fast as possible. + + On entry to this basic block: + t1 == the shifted high-order bits from the previous source word + t2 == the unshifted current source word + + We further know that t2 does not contain a null terminator. */ + + .align 4 +$u_loop: + extqh t2, a1, t0 # U : extract high bits for current word + addq a1, 8, a1 # E : + extql t2, a1, t3 # U : extract low bits for next time + addq a0, 8, a0 # E : + + or t0, t1, t0 # E : current dst word now complete + ldq_u t2, 0(a1) # U : Latency=3 load high word for next time + stq_u t0, -8(a0) # U : save the current word (stall) + mov t3, t1 # E : + + subq a2, 1, a2 # E : + cmpbge zero, t2, t8 # E : test new word for eos (2 cycle stall for data) + beq a2, $u_eoc # U : (stall) + nop + + beq t8, $u_loop # U : + nop + nop + nop + + /* We've found a zero somewhere in the source word we just read. + If it resides in the lower half, we have one (probably partial) + word to write out, and if it resides in the upper half, we + have one full and one partial word left to write out. + + On entry to this basic block: + t1 == the shifted high-order bits from the previous source word + t2 == the unshifted current source word. */ +$u_eos: + extqh t2, a1, t0 # U : + or t0, t1, t0 # E : first (partial) source word complete (stall) + cmpbge zero, t0, t8 # E : is the null in this first bit? (stall) + bne t8, $u_final # U : (stall) + + stq_u t0, 0(a0) # L : the null was in the high-order bits + addq a0, 8, a0 # E : + subq a2, 1, a2 # E : + nop + +$u_late_head_exit: + extql t2, a1, t0 # U : + cmpbge zero, t0, t8 # E : + or t8, t10, t6 # E : (stall) + cmoveq a2, t6, t8 # E : Latency=2, extra map slot (stall) + + /* Take care of a final (probably partial) result word. + On entry to this basic block: + t0 == assembled source word + t8 == cmpbge mask that found the null. */ +$u_final: + negq t8, t6 # E : isolate low bit set + and t6, t8, t12 # E : (stall) + and t12, 0x80, t6 # E : avoid dest word load if we can (stall) + bne t6, 1f # U : (stall) + + ldq_u t1, 0(a0) # L : + subq t12, 1, t6 # E : + or t6, t12, t8 # E : (stall) + zapnot t0, t8, t0 # U : kill source bytes > null + + zap t1, t8, t1 # U : kill dest bytes <= null + or t0, t1, t0 # E : (stall) + nop + nop + +1: stq_u t0, 0(a0) # L : + ret (t9) # L0 : Latency=3 + +$u_eoc: # end-of-count + extqh t2, a1, t0 # U : + or t0, t1, t0 # E : (stall) + cmpbge zero, t0, t8 # E : (stall) + nop + +$u_eocfin: # end-of-count, final word + or t10, t8, t8 # E : + br $u_final # L0 : Latency=3 + nop + nop + + /* Unaligned copy entry point. */ + .align 4 +$unaligned: + + ldq_u t1, 0(a1) # L : load first source word + and a0, 7, t4 # E : find dest misalignment + and a1, 7, t5 # E : find src misalignment + /* Conditionally load the first destination word and a bytemask + with 0xff indicating that the destination byte is sacrosanct. */ + mov zero, t0 # E : + + mov zero, t6 # E : + beq t4, 1f # U : + ldq_u t0, 0(a0) # L : + lda t6, -1 # E : + + mskql t6, a0, t6 # U : + nop + nop + nop +1: + subq a1, t4, a1 # E : sub dest misalignment from src addr + + /* If source misalignment is larger than dest misalignment, we need + extra startup checks to avoid SEGV. */ + + cmplt t4, t5, t12 # E : + extql t1, a1, t1 # U : shift src into place + lda t2, -1 # E : for creating masks later + beq t12, $u_head # U : (stall) + + mskqh t2, t5, t2 # U : begin src byte validity mask + cmpbge zero, t1, t8 # E : is there a zero? + extql t2, a1, t2 # U : + or t8, t10, t5 # E : test for end-of-count too + + cmpbge zero, t2, t3 # E : + cmoveq a2, t5, t8 # E : Latency=2, extra map slot + nop # E : keep with cmoveq + andnot t8, t3, t8 # E : (stall) + + beq t8, $u_head # U : + /* At this point we've found a zero in the first partial word of + the source. We need to isolate the valid source data and mask + it into the original destination data. (Incidentally, we know + that we'll need at least one byte of that original dest word.) */ + ldq_u t0, 0(a0) # L : + negq t8, t6 # E : build bitmask of bytes <= zero + mskqh t1, t4, t1 # U : + + and t6, t8, t12 # E : + subq t12, 1, t6 # E : (stall) + or t6, t12, t8 # E : (stall) + zapnot t2, t8, t2 # U : prepare source word; mirror changes (stall) + + zapnot t1, t8, t1 # U : to source validity mask + andnot t0, t2, t0 # E : zero place for source to reside + or t0, t1, t0 # E : and put it there (stall both t0, t1) + stq_u t0, 0(a0) # L : (stall) + + ret (t9) # L0 : Latency=3 + nop + nop + nop + + .end __stxncpy + +#endif diff -Naur linux-2.4.0-test9.ref/include/asm-alpha/bitops.h linux-2.4.0-test9-perfpatch/include/asm-alpha/bitops.h --- linux-2.4.0-test9.ref/include/asm-alpha/bitops.h Tue Oct 3 12:24:41 2000 +++ linux-2.4.0-test9-perfpatch/include/asm-alpha/bitops.h Tue Nov 14 09:34:57 2000 @@ -279,19 +279,39 @@ unsigned long sum = 0; x = ~x & -~x; /* set first 0 bit, clear others */ +#if (defined(__alpha_cix__) || defined(__alpha_fix__)) && defined(CONFIG_ALPHA_EV67) + + /* + * Count bits set in a byte + * If the bits were all zeros, cttz comes back with 0x40. + * Correct for that case. + */ + x &= 0xff; + __asm__ __volatile__ ("cttz %1,%0":"=r" (sum):"r"(x)); + sum &= 0xf; +#else if (x & 0xF0) sum += 4; if (x & 0xCC) sum += 2; if (x & 0xAA) sum += 1; +#endif return sum; } extern inline unsigned long ffz(unsigned long word) { -#if defined(__alpha_cix__) && defined(__alpha_fix__) +#if (defined(__alpha_cix__) || defined(__alpha_fix__)) && defined(CONFIG_ALPHA_EV67) + /* Whee. EV6 can calculate it directly. */ unsigned long result; - __asm__("cttz %1,%0" : "=r"(result) : "r"(~word)); + __asm__ __volatile__("cttz %1,%0" : "=r"(result) : "r"(~word)); + + /* + * If the bits were all zeros, cttz comes back with 0x40. + * Correct for that case. + */ + result &= 0x3f; + return result; #else unsigned long bits, qofs, bofs; @@ -315,7 +335,14 @@ extern inline int ffs(int word) { - int result = ffz(~word); + int result; + +#if (defined(__alpha_cix__) || defined(__alpha_fix__)) && defined(CONFIG_ALPHA_EV67) + + __asm__ __volatile__ ("cttz %1,%0":"=r" (result):"r"(word)); +#else + result = ffz(~word); +#endif return word ? result+1 : 0; } @@ -324,7 +351,8 @@ * of bits set) of a N-bit word */ -#if defined(__alpha_cix__) && defined(__alpha_fix__) +#if (defined(__alpha_cix__) || defined(__alpha_fix__)) && defined(CONFIG_ALPHA_EV67) + /* Whee. EV6 can calculate it directly. */ extern __inline__ unsigned long hweight64(unsigned long w) { diff -Naur linux-2.4.0-test9.ref/include/asm-alpha/byteorder.h linux-2.4.0-test9-perfpatch/include/asm-alpha/byteorder.h --- linux-2.4.0-test9.ref/include/asm-alpha/byteorder.h Sat Jun 6 21:52:04 1998 +++ linux-2.4.0-test9-perfpatch/include/asm-alpha/byteorder.h Mon Oct 9 10:52:55 2000 @@ -11,6 +11,21 @@ static __inline__ __const__ __u32 ___arch__swab32(__u32 x) { +/* + * Unfortunately, we can't use the 6 instruction sequence + * on ev6 since the latency of the UNPKBW is 3, which is + * pretty hard to hide. Just in case a future implementation + * has a lower latency, here's the sequence (also by Mike Burrows) + * + * UNPKBW a0, v0 v0: 00AA00BB00CC00DD + * SLL v0, 24, a0 a0: BB00CC00DD000000 + * BIS v0, a0, a0 a0: BBAACCBBDDCC00DD + * EXTWL a0, 6, v0 v0: 000000000000BBAA + * ZAP a0, 0xf3, a0 a0: 00000000DDCC0000 + * ADDL a0, v0, v0 v0: ssssssssDDCCBBAA + */ + +#if 0 __u64 t1, t2, t3; /* Break the final or's out of the block so that gcc can @@ -30,6 +45,25 @@ : "r"(x)); return t3 + t2 + t1; +#else + + __u64 t1, t2, t3, t4; + + /* 7 instruction sequence (by Mike Burrows) */ + + __asm__( + "INSLH %4, 7, %0 # %0 : 0000000000AABBCC\n\t" + "INSWL %4, 3, %1 # %1 : 000000CCDD000000\n\t" + "BIS %1, %0, %1 # %1 : 000000CCDDAABBCC\n\t" + "SRL %1, 16, %2 # %2 : 0000000000CCDDAA\n\t" + "ZAPNOT %1, 0xa, %0 # %0 : 00000000DD00BB00\n\t" + "ZAP %2, 0xa, %3 # %3 : 0000000000CC00AA\n\t" + "ADDL %3, %0, %1 # %2 : ssssssssDDCCBBAA\n\t" + : "=r"(t3), "=&r"(t1), "=&r"(t2), "=&r"(t4) + : "r"(x)); + + return t1; +#endif } static __inline__ __const__ __u16 ___arch__swab16(__u16 x) diff -Naur linux-2.4.0-test9.ref/include/asm-alpha/page.h linux-2.4.0-test9-perfpatch/include/asm-alpha/page.h --- linux-2.4.0-test9.ref/include/asm-alpha/page.h Wed Aug 9 16:46:01 2000 +++ linux-2.4.0-test9-perfpatch/include/asm-alpha/page.h Mon Oct 9 10:52:55 2000 @@ -1,6 +1,7 @@ #ifndef _ALPHA_PAGE_H #define _ALPHA_PAGE_H + /* PAGE_SHIFT determines the page size */ #define PAGE_SHIFT 13 #define PAGE_SIZE (1UL << PAGE_SHIFT) @@ -21,10 +22,26 @@ */ static inline void clear_page(void * page) { - unsigned long count = PAGE_SIZE/64; + unsigned long count = (PAGE_SIZE - 256)/64; unsigned long *ptr = (unsigned long *)page; + unsigned long *temp; do { + temp = ptr + 32; + __asm__("wh64 (%0)" : "+r" (temp)); + temp[-32] = 0; /* prevents removal of wh64 insn */ + ptr[1] = 0; + ptr[2] = 0; + ptr[3] = 0; + count--; + ptr[4] = 0; + ptr[5] = 0; + ptr[6] = 0; + ptr[7] = 0; + ptr += 8; + } while (count); + count += 4; + do { ptr[0] = 0; ptr[1] = 0; ptr[2] = 0; @@ -42,10 +59,37 @@ static inline void copy_page(void * _to, void * _from) { - unsigned long count = PAGE_SIZE/64; + unsigned long count = (PAGE_SIZE - 256)/64; unsigned long *to = (unsigned long *)_to; unsigned long *from = (unsigned long *)_from; + unsigned long *temp; + do { + unsigned long a,b,c,d,e,f,g,h; + temp = to+32; + __asm__("wh64 (%0)" : "+r" (temp)); + __asm__("ldl $31, 256(%0)" : "+r" (from)); + a = from[0]; + b = from[1]; + c = from[2]; + d = from[3]; + e = from[4]; + f = from[5]; + g = from[6]; + h = from[7]; + count--; + from += 8; + temp[-32] = a; /*prevents removal of wh64 insn */ + to[1] = b; + to[2] = c; + to[3] = d; + to[4] = e; + to[5] = f; + to[6] = g; + to[7] = h; + to += 8; + } while (count); + count += 4; do { unsigned long a,b,c,d,e,f,g,h; a = from[0]; diff -Naur linux-2.4.0-test9.ref/include/asm-alpha/pgtable.h linux-2.4.0-test9-perfpatch/include/asm-alpha/pgtable.h --- linux-2.4.0-test9.ref/include/asm-alpha/pgtable.h Tue Aug 8 00:02:27 2000 +++ linux-2.4.0-test9-perfpatch/include/asm-alpha/pgtable.h Mon Oct 9 12:38:44 2000 @@ -58,6 +58,11 @@ #define _PAGE_FOW 0x0004 /* used for page protection (fault on write) */ #define _PAGE_FOE 0x0008 /* used for page protection (fault on exec) */ #define _PAGE_ASM 0x0010 +#if defined(CONFIG_ALPHA_EV6) && !defined(__SMP__) +#define _PAGE_MBE 0x0080 /* MB disable bit for EV6 */ +#else +#define _PAGE_MBE 0x0000 +#endif #define _PAGE_KRE 0x0100 /* xxx - see below on the "accessed" bit */ #define _PAGE_URE 0x0200 /* xxx */ #define _PAGE_KWE 0x1000 /* used to do the dirty bit in software */ @@ -193,7 +198,7 @@ pte_t pte; \ \ pte_val(pte) = ((unsigned long)(page - mem_map) << 32) | \ - pgprot_val(pgprot); \ + pgprot_val(pgprot)|_PAGE_MBE; \ pte; \ })