| ; Copyright © 2018, VideoLAN and dav1d authors |
| ; Copyright © 2018, Two Orioles, LLC |
| ; All rights reserved. |
| ; |
| ; Redistribution and use in source and binary forms, with or without |
| ; modification, are permitted provided that the following conditions are met: |
| ; |
| ; 1. Redistributions of source code must retain the above copyright notice, this |
| ; list of conditions and the following disclaimer. |
| ; |
| ; 2. Redistributions in binary form must reproduce the above copyright notice, |
| ; this list of conditions and the following disclaimer in the documentation |
| ; and/or other materials provided with the distribution. |
| ; |
| ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
| ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR |
| ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| %include "config.asm" |
| %undef private_prefix |
| %define private_prefix checkasm |
| %include "ext/x86/x86inc.asm" |
| |
| SECTION_RODATA 16 |
| |
| %if ARCH_X86_64 |
| ; just random numbers to reduce the chance of incidental match |
| %if WIN64 |
| x6: dq 0x1a1b2550a612b48c,0x79445c159ce79064 |
| x7: dq 0x2eed899d5a28ddcd,0x86b2536fcd8cf636 |
| x8: dq 0xb0856806085e7943,0x3f2bf84fc0fcca4e |
| x9: dq 0xacbd382dcf5b8de2,0xd229e1f5b281303f |
| x10: dq 0x71aeaff20b095fd9,0xab63e2e11fa38ed9 |
| x11: dq 0x89b0c0765892729a,0x77d410d5c42c882d |
| x12: dq 0xc45ea11a955d8dd5,0x24b3c1d2a024048b |
| x13: dq 0x2e8ec680de14b47c,0xdd7b8919edd42786 |
| x14: dq 0x135ce6888fa02cbf,0x11e53e2b2ac655ef |
| x15: dq 0x011ff554472a7a10,0x6de8f4c914c334d5 |
| n7: dq 0x21f86d66c8ca00ce |
| n8: dq 0x75b6ba21077c48ad |
| %endif |
| n9: dq 0xed56bb2dcb3c7736 |
| n10: dq 0x8bda43d3fd1a7e06 |
| n11: dq 0xb64a9c9e5d318408 |
| n12: dq 0xdf9a54b303f1d3a3 |
| n13: dq 0x4a75479abd64e097 |
| n14: dq 0x249214109d5d1c88 |
| %endif |
| |
| errmsg_stack: db "stack corruption", 0 |
| errmsg_register: db "failed to preserve register:%s", 0 |
| errmsg_vzeroupper: db "missing vzeroupper", 0 |
| |
| SECTION .bss |
| |
| check_vzeroupper: resd 1 |
| |
| SECTION .text |
| |
| cextern fail_func |
| |
| ; max number of args used by any asm function. |
| ; (max_args % 4) must equal 3 for stack alignment |
| %define max_args 15 |
| |
| %if UNIX64 |
| DECLARE_REG_TMP 0 |
| %else |
| DECLARE_REG_TMP 4 |
| %endif |
| |
| ;----------------------------------------------------------------------------- |
| ; unsigned checkasm_init_x86(char *name) |
| ;----------------------------------------------------------------------------- |
| cglobal init_x86, 0, 5 |
| %if ARCH_X86_64 |
| push rbx |
| %endif |
| movifnidn t0, r0mp |
| mov eax, 0x80000000 |
| cpuid |
| cmp eax, 0x80000004 |
| jb .no_brand ; processor brand string not supported |
| mov eax, 0x80000002 |
| cpuid |
| mov [t0+4* 0], eax |
| mov [t0+4* 1], ebx |
| mov [t0+4* 2], ecx |
| mov [t0+4* 3], edx |
| mov eax, 0x80000003 |
| cpuid |
| mov [t0+4* 4], eax |
| mov [t0+4* 5], ebx |
| mov [t0+4* 6], ecx |
| mov [t0+4* 7], edx |
| mov eax, 0x80000004 |
| cpuid |
| mov [t0+4* 8], eax |
| mov [t0+4* 9], ebx |
| mov [t0+4*10], ecx |
| mov [t0+4*11], edx |
| xor eax, eax |
| cpuid |
| jmp .check_xcr1 |
| .no_brand: ; use manufacturer id as a fallback |
| xor eax, eax |
| mov [t0+4*3], eax |
| cpuid |
| mov [t0+4*0], ebx |
| mov [t0+4*1], edx |
| mov [t0+4*2], ecx |
| .check_xcr1: |
| test eax, eax |
| jz .end2 ; cpuid leaf 1 not supported |
| mov t0d, eax ; max leaf |
| mov eax, 1 |
| cpuid |
| and ecx, 0x18000000 |
| cmp ecx, 0x18000000 |
| jne .end2 ; osxsave/avx not supported |
| cmp t0d, 13 ; cpuid leaf 13 not supported |
| jb .end2 |
| mov t0d, eax ; cpuid signature |
| mov eax, 13 |
| mov ecx, 1 |
| cpuid |
| test al, 0x04 |
| jz .end ; xcr1 not supported |
| mov ecx, 1 |
| xgetbv |
| test al, 0x04 |
| jnz .end ; always-dirty ymm state |
| %if ARCH_X86_64 == 0 && PIC |
| LEA eax, check_vzeroupper |
| mov [eax], ecx |
| %else |
| mov [check_vzeroupper], ecx |
| %endif |
| .end: |
| mov eax, t0d |
| .end2: |
| %if ARCH_X86_64 |
| pop rbx |
| %endif |
| RET |
| |
| %if ARCH_X86_64 |
| %if WIN64 |
| %define stack_param rsp+32 ; shadow space |
| %define num_fn_args rsp+stack_offset+17*8 |
| %assign num_reg_args 4 |
| %assign free_regs 7 |
| %assign clobber_mask_stack_bit 16 |
| DECLARE_REG_TMP 4 |
| %else |
| %define stack_param rsp |
| %define num_fn_args rsp+stack_offset+11*8 |
| %assign num_reg_args 6 |
| %assign free_regs 9 |
| %assign clobber_mask_stack_bit 64 |
| DECLARE_REG_TMP 7 |
| %endif |
| |
| %macro CLOBBER_UPPER 2 ; reg, mask_bit |
| mov r13d, %1d |
| or r13, r8 |
| test r9b, %2 |
| cmovnz %1, r13 |
| %endmacro |
| |
| cglobal checked_call, 2, 15, 16, max_args*8+64+8 |
| mov r10d, [num_fn_args] |
| mov r8, 0xdeadbeef00000000 |
| mov r9d, [num_fn_args+r10*8+8] ; clobber_mask |
| mov t0, [num_fn_args+r10*8] ; func |
| |
| ; Clobber the upper halves of 32-bit parameters |
| CLOBBER_UPPER r0, 1 |
| CLOBBER_UPPER r1, 2 |
| CLOBBER_UPPER r2, 4 |
| CLOBBER_UPPER r3, 8 |
| %if UNIX64 |
| CLOBBER_UPPER r4, 16 |
| CLOBBER_UPPER r5, 32 |
| %else ; WIN64 |
| %assign i 6 |
| %rep 16-6 |
| mova m %+ i, [x %+ i] |
| %assign i i+1 |
| %endrep |
| %endif |
| |
| xor r11d, r11d |
| sub r10d, num_reg_args |
| cmovs r10d, r11d ; num stack args |
| |
| ; write stack canaries to the area above parameters passed on the stack |
| mov r12, [rsp+stack_offset] ; return address |
| not r12 |
| %assign i 0 |
| %rep 8 ; 64 bytes |
| mov [stack_param+(r10+i)*8], r12 |
| %assign i i+1 |
| %endrep |
| |
| test r10d, r10d |
| jz .stack_setup_done ; no stack parameters |
| .copy_stack_parameter: |
| mov r12, [stack_param+stack_offset+8+r11*8] |
| CLOBBER_UPPER r12, clobber_mask_stack_bit |
| shr r9d, 1 |
| mov [stack_param+r11*8], r12 |
| inc r11d |
| cmp r11d, r10d |
| jl .copy_stack_parameter |
| .stack_setup_done: |
| |
| %assign i 14 |
| %rep 15-free_regs |
| mov r %+ i, [n %+ i] |
| %assign i i-1 |
| %endrep |
| call t0 |
| |
| ; check for stack corruption |
| mov r0d, [num_fn_args] |
| xor r3d, r3d |
| sub r0d, num_reg_args |
| cmovs r0d, r3d ; num stack args |
| |
| mov r3, [rsp+stack_offset] |
| mov r4, [stack_param+r0*8] |
| not r3 |
| xor r4, r3 |
| %assign i 1 |
| %rep 6 |
| mov r5, [stack_param+(r0+i)*8] |
| xor r5, r3 |
| or r4, r5 |
| %assign i i+1 |
| %endrep |
| xor r3, [stack_param+(r0+7)*8] |
| or r4, r3 |
| jz .stack_ok |
| ; Save the return value located in rdx:rax first to prevent clobbering. |
| mov r10, rax |
| mov r11, rdx |
| lea r0, [errmsg_stack] |
| jmp .fail |
| .stack_ok: |
| |
| ; check for failure to preserve registers |
| %assign i 14 |
| %rep 15-free_regs |
| cmp r %+ i, [n %+ i] |
| setne r4b |
| lea r3d, [r4+r3*2] |
| %assign i i-1 |
| %endrep |
| %if WIN64 |
| lea r0, [rsp+32] ; account for shadow space |
| mov r5, r0 |
| test r3d, r3d |
| jz .gpr_ok |
| %else |
| test r3d, r3d |
| jz .gpr_xmm_ok |
| mov r0, rsp |
| %endif |
| %assign i free_regs |
| %rep 15-free_regs |
| %if i < 10 |
| mov dword [r0], " r0" + (i << 16) |
| lea r4, [r0+3] |
| %else |
| mov dword [r0], " r10" + ((i - 10) << 24) |
| lea r4, [r0+4] |
| %endif |
| test r3b, 1 << (i - free_regs) |
| cmovnz r0, r4 |
| %assign i i+1 |
| %endrep |
| %if WIN64 ; xmm registers |
| .gpr_ok: |
| %assign i 6 |
| %rep 16-6 |
| pxor m %+ i, [x %+ i] |
| %assign i i+1 |
| %endrep |
| packsswb m6, m7 |
| packsswb m8, m9 |
| packsswb m10, m11 |
| packsswb m12, m13 |
| packsswb m14, m15 |
| packsswb m6, m6 |
| packsswb m8, m10 |
| packsswb m12, m14 |
| packsswb m6, m6 |
| packsswb m8, m12 |
| packsswb m6, m8 |
| pxor m7, m7 |
| pcmpeqb m6, m7 |
| pmovmskb r3d, m6 |
| cmp r3d, 0xffff |
| je .xmm_ok |
| mov r7d, " xmm" |
| %assign i 6 |
| %rep 16-6 |
| mov [r0+0], r7d |
| %if i < 10 |
| mov byte [r0+4], "0" + i |
| lea r4, [r0+5] |
| %else |
| mov word [r0+4], "10" + ((i - 10) << 8) |
| lea r4, [r0+6] |
| %endif |
| test r3d, 1 << i |
| cmovz r0, r4 |
| %assign i i+1 |
| %endrep |
| .xmm_ok: |
| cmp r0, r5 |
| je .gpr_xmm_ok |
| mov byte [r0], 0 |
| mov r11, rdx |
| mov r1, r5 |
| %else |
| mov byte [r0], 0 |
| mov r11, rdx |
| mov r1, rsp |
| %endif |
| mov r10, rax |
| lea r0, [errmsg_register] |
| jmp .fail |
| .gpr_xmm_ok: |
| ; Check for dirty YMM state, i.e. missing vzeroupper |
| mov ecx, [check_vzeroupper] |
| test ecx, ecx |
| jz .ok ; not supported, skip |
| mov r10, rax |
| mov r11, rdx |
| xgetbv |
| test al, 0x04 |
| jz .restore_retval ; clean ymm state |
| lea r0, [errmsg_vzeroupper] |
| vzeroupper |
| .fail: |
| ; Call fail_func() with a descriptive message to mark it as a failure. |
| xor eax, eax |
| call fail_func |
| .restore_retval: |
| mov rax, r10 |
| mov rdx, r11 |
| .ok: |
| RET |
| |
| ; trigger a warmup of vector units |
| %macro WARMUP 0 |
| cglobal warmup, 0, 0 |
| xorps m0, m0 |
| mulps m0, m0 |
| RET |
| %endmacro |
| |
| INIT_YMM avx2 |
| WARMUP |
| INIT_ZMM avx512 |
| WARMUP |
| |
| %else |
| |
| ; just random numbers to reduce the chance of incidental match |
| %assign n3 0x6549315c |
| %assign n4 0xe02f3e23 |
| %assign n5 0xb78d0d1d |
| %assign n6 0x33627ba7 |
| |
| ;----------------------------------------------------------------------------- |
| ; void checkasm_checked_call(void *func, ...) |
| ;----------------------------------------------------------------------------- |
| cglobal checked_call, 1, 7 |
| mov r3, [esp+stack_offset] ; return address |
| mov r1, [esp+stack_offset+17*4] ; num_stack_params |
| mov r2, 27 |
| not r3 |
| sub r2, r1 |
| .push_canary: |
| push r3 |
| dec r2 |
| jg .push_canary |
| .push_parameter: |
| push dword [esp+32*4] |
| dec r1 |
| jg .push_parameter |
| mov r3, n3 |
| mov r4, n4 |
| mov r5, n5 |
| mov r6, n6 |
| call r0 |
| |
| ; check for failure to preserve registers |
| cmp r3, n3 |
| setne r3h |
| cmp r4, n4 |
| setne r3b |
| shl r3d, 16 |
| cmp r5, n5 |
| setne r3h |
| cmp r6, n6 |
| setne r3b |
| test r3, r3 |
| jz .gpr_ok |
| lea r1, [esp+16] |
| mov [esp+4], r1 |
| %assign i 3 |
| %rep 4 |
| mov dword [r1], " r0" + (i << 16) |
| lea r4, [r1+3] |
| test r3, 1 << ((6 - i) * 8) |
| cmovnz r1, r4 |
| %assign i i+1 |
| %endrep |
| mov byte [r1], 0 |
| mov r5, eax |
| mov r6, edx |
| LEA r1, errmsg_register |
| jmp .fail |
| .gpr_ok: |
| ; check for stack corruption |
| mov r3, [esp+48*4] ; num_stack_params |
| mov r6, [esp+31*4] ; return address |
| mov r4, [esp+r3*4] |
| sub r3, 26 |
| not r6 |
| xor r4, r6 |
| .check_canary: |
| mov r5, [esp+(r3+27)*4] |
| xor r5, r6 |
| or r4, r5 |
| inc r3 |
| jl .check_canary |
| mov r5, eax |
| mov r6, edx |
| test r4, r4 |
| jz .stack_ok |
| LEA r1, errmsg_stack |
| jmp .fail |
| .stack_ok: |
| ; check for dirty YMM state, i.e. missing vzeroupper |
| LEA ecx, check_vzeroupper |
| mov ecx, [ecx] |
| test ecx, ecx |
| jz .ok ; not supported, skip |
| xgetbv |
| test al, 0x04 |
| jz .ok ; clean ymm state |
| LEA r1, errmsg_vzeroupper |
| vzeroupper |
| .fail: |
| mov [esp], r1 |
| call fail_func |
| .ok: |
| add esp, 27*4 |
| mov eax, r5 |
| mov edx, r6 |
| RET |
| |
| %endif ; ARCH_X86_64 |
