procyberian/nasm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

BR 2028910: fix decoding of VEX prefixes in 16- and 32-bit mode

Browse source 
We would incorrectly set a bunch of VEX-related state for C4 and C5
bytes, even though we had already rejected it as not a VEX prefix due
to the top two bits of the following byte not being 11.
This commit is contained in:
H. Peter Anvin 2008-07-30 17:30:12 -07:00
parent ecf8c3e382
commit f7d863b7d1
1 changed files with 18 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
disasm.c
View file

@ -1073,24 +1073,25 @@ int32_t disasm(uint8_t *data, char *output, int outbufsize, int segsize,
if (segsize == 64 || (data[1] & 0xc0) == 0xc0) {
prefix.vex[0] = *data++;
prefix.vex[1] = *data++;
if (prefix.vex[0] == 0xc4)
prefix.vex[2] = *data++;
}
prefix.rex = REX_V;
if (prefix.vex[0] == 0xc4) {
prefix.rex |= (~prefix.vex[1] >> 5) & 7; /* REX_RXB */
prefix.rex |= (prefix.vex[2] >> (7-3)) & REX_W;
prefix.vex_m = prefix.vex[1] & 0x1f;
prefix.vex_v = (~prefix.vex[2] >> 3) & 15;
prefix.vex_lp = prefix.vex[2] & 7;
} else {
prefix.rex |= (~prefix.vex[1] >> (7-2)) & REX_R;
prefix.vex_m = 1;
prefix.vex_v = (~prefix.vex[1] >> 3) & 15;
prefix.vex_lp = prefix.vex[1] & 7;
}
ix = itable_VEX[prefix.vex_m][prefix.vex_lp];
prefix.rex = REX_V;
if (prefix.vex[0] == 0xc4) {
prefix.vex[2] = *data++;
prefix.rex |= (~prefix.vex[1] >> 5) & 7; /* REX_RXB */
prefix.rex |= (prefix.vex[2] >> (7-3)) & REX_W;
prefix.vex_m = prefix.vex[1] & 0x1f;
prefix.vex_v = (~prefix.vex[2] >> 3) & 15;
prefix.vex_lp = prefix.vex[2] & 7;
} else {
prefix.rex |= (~prefix.vex[1] >> (7-2)) & REX_R;
prefix.vex_m = 1;
prefix.vex_v = (~prefix.vex[1] >> 3) & 15;
prefix.vex_lp = prefix.vex[1] & 7;
}
ix = itable_VEX[prefix.vex_m][prefix.vex_lp];
}
end_prefix = true;
break;