**Description**

> Please submit RCTF{\<WhatYouInput\>}.
>
> attachment: https://drive.google.com/open?id=1AQykwr6bNklqdtaVkFs0B79QAyrixXnd

**Solution**

As the name indicates, the core of this challenge was a virtual machine with custom opcodes and emulation:

```c
__int64 sub_400896()
{//code of vm engine
__int64 instr_pointer; // rax
_BYTE *vm_code; // rbp
int nxt_instr_pointer; // ebx
__int64 v4; // rdx
__int64 v5; // rax
__int64 v6; // rax
__int64 v7; // rax
__int64 v8; // rax
__int64 v9; // rax
int v10; // eax
__int64 v11; // rax
char v12; // dl
int v13; // eax
int v14; // eax
_BYTE *v15; // rax
__int64 v16; // rax
__int64 v17; // rax
__int64 v18; // rax

instr_pointer = 0LL;
vm_code = ::vm_code;
while ( 1 )
{
nxt_instr_pointer = instr_pointer + 1;
switch ( vm_code[instr_pointer] )
{
case 0:
return *(unsigned int *)&vm_code[nxt_instr_pointer];
case 1:
goto LABEL_35;
case 2:
v4 = nxt_instr_pointer;
nxt_instr_pointer = instr_pointer + 9;
vm_code[*(signed int *)&vm_code[v4]] = *(_DWORD *)&vm_code[(signed int)instr_pointer + 5];
break;
case 3:
v5 = nxt_instr_pointer;
nxt_instr_pointer += 4;
v6 = *(signed int *)&vm_code[v5];
goto LABEL_27;
case 4:
v7 = nxt_instr_pointer;
nxt_instr_pointer += 4;
v8 = *(signed int *)&vm_code[v7];
goto LABEL_31;
case 5:
v9 = nxt_instr_pointer;
nxt_instr_pointer += 4;
v10 = (char)vm_code[*(signed int *)&vm_code[v9]];
goto LABEL_21;
case 6:
v11 = nxt_instr_pointer;
v12 = g_var;
nxt_instr_pointer += 4;
v8 = *(signed int *)&vm_code[v11];
goto LABEL_9;
case 7:
v13 = g_var;
goto LABEL_23;
case 8:
v14 = ~(g_var & condition);
goto LABEL_12;
case 0xA:
v14 = getchar();
goto LABEL_12;
case 0xB:
putchar(condition);
break;
case 0xC:
v15 = &vm_code[*(signed int *)&vm_code[nxt_instr_pointer]];
if ( *v15 )
{
nxt_instr_pointer = *(_DWORD *)&vm_code[nxt_instr_pointer + 4];
--*v15;
}
else
{
nxt_instr_pointer += 8;
}
break;
case 0xD:
++condition;
break;
case 0xE:
++g_var;
break;
case 0xF:
v14 = g_var;
goto LABEL_12;
case 0x10:
v10 = condition;
goto LABEL_21;
case 0x11:
v16 = nxt_instr_pointer;
nxt_instr_pointer += 4;
v13 = *(_DWORD *)&vm_code[v16];
LABEL_23:
condition += v13;
break;
case 0x12:
v6 = g_var;
goto LABEL_27;
case 0x13:
v6 = condition;
LABEL_27:
v14 = (char)vm_code[v6];
goto LABEL_12;
case 0x14:
v17 = nxt_instr_pointer;
nxt_instr_pointer += 4;
v14 = *(_DWORD *)&vm_code[v17];
goto LABEL_12;
case 0x15:
v18 = nxt_instr_pointer;
nxt_instr_pointer += 4;
v10 = *(_DWORD *)&vm_code[v18];
LABEL_21:
g_var = v10;
break;
case 0x16:
v8 = g_var;
LABEL_31:
v12 = condition;
LABEL_9:
vm_code[v8] = v12;
break;
case 0x17:
v14 = condition - g_var;
LABEL_12:
condition = v14;
break;
case 0x18:
if ( condition )
LABEL_35:
nxt_instr_pointer = *(_DWORD *)&vm_code[nxt_instr_pointer];
else
nxt_instr_pointer = instr_pointer + 5;
break;
default:
break;
}
if ( nxt_instr_pointer >= file_size )
return 0LL;
instr_pointer = nxt_instr_pointer;
}
}
```

I wrote a Python script to step through the VM bytecode and analyse each step.

([full script](https://github.com/Aurel300/empirectf/blob/master/writeups/2018-05-19-RCTF/scripts/simple-vm.py))

The output was:

```c
0x30: global = 0x100
0x35: global += 1
0x36: cond = byte [global]
0x37: putchar(cond)
0x38: byte [0x100] ? jmp 0x35 and --
0x41: nop

// Output "Input Flag:"
0x42: global = 0x110
0x47: global += 1
0x48: cond = getchar()
0x49: nop
0x4a: [global] = (byte)cond
0x4b: byte [0x110] ? jmp 0x47 and --
// [0x110] == 0x1f, flag size is 32
// Input stored in [0x111]

0x54: nop

// Do some bit operation to each character
0x55: cond = byte [0x140] //[0x140] == 0x20 + i
0x5a: global = cond
0x5b: cond += 0xf1 // Cond points to input
0x60: cond = byte [cond]
// Cond = input[i], [0x111]
0x61: [0x143] = (byte)cond
0x66: cond = ~(global & cond)
0x67: [0x141] = (byte)cond
0x6c: global = cond
0x6d: cond = byte [0x140]
0x72: cond = ~(global & cond)
0x73: [0x142] = (byte)cond
0x78: cond = byte [0x141]
0x7d: cond = byte [0x143]
0x82: cond = ~(global & cond)
0x83: global = cond
0x84: cond = byte [0x142]
0x89: cond = ~(global & cond)
0x8a: [0x144] = (byte)cond
0x8f: nop
0x90: cond = byte [0x140]
0x95: cond += 0xf1
0x9a: global = cond
0x9b: cond = byte [0x144]
0xa0: [global] = (byte)cond
0xa1: global = byte [0x140]
0xa6: global += 1
0xa7: [0x140] = global
0xac: byte [0x145] ? jmp 0x55 and --

// Compare with key
0xb5: nop
0xb6: cond = byte [0x146] //[0x146] is i
0xbb: cond += 0x5 //5[i]
0xc0: cond = byte [cond] //take 5[i]
0xc1: global = cond //global = 5[i], which is key
0xc2: cond = byte [0x146]
0xc7: cond += 0x111
0xcc: cond = byte [cond] // 0x111[i] == 5[i], 0x111[i] is the result of bit operation
0xcd: cond -= global
0xce: cond ? jmp 0x160
0xd3: byte [0x146] ? jmp 0xb6 and --
0xdc: jmp 0x176 // Success
0xe1: nop
```

`global` and `cond` are actually registers. Care has to be taken with byte access or dword access, and signed extension or unsigned extension (`movzx` or `movsx`), which are likely to be handled mistakenly.

Obtaining the vmcode and understanding what it does, we can write a bruteforce crack:

```python
keys = [0x10,0x18,0x43,0x14,0x15,0x47,0x40,0x17,0x10,0x1d,0x4b,
0x12,0x1f,0x49,0x48,0x18,0x53,0x54,0x01,0x57,0x51,0x53,0x05,0x56,0x5a,0x08,0x58,
0x5f,0x0a,0x0c,0x58,0x09]

def some_bit_oper(c, i):
i += 0x20
g = i
t3 = c
t1 = (~(i & c)) & 0xff
t2 = (~(t1 & i)) & 0xff
t4 = (~((~(t1 & t3)) & t2)) & 0xff
return t4

flag = []
for i in xrange(0, 32):
for c in xrange(0, 256):
if some_bit_oper(c, i) == keys[i]:
flag.append(chr(c))

print "".join(flag)
```

`RCTF{09a71bf084a93df7ce3def3ab1bd61f6}`