[PATCH] ncrewrite, a tool for rewriting masking instructions before indirect jumps
Mark Seaborn
library that works under NaCl to one that works with the native Linux
glibc, and back again.
ncrewrite rewrites the alignment instructions that nacl-gcc inserts
before indirect jumps into no-ops, so
and $0xffffffe0, %reg
jmp *%reg
becomes
nop; nop; nop
jmp *%reg
The resulting executable will then work with libraries where indirect
jump targets (i.e. function entry points and function call return
addresses) are not 32-byte-aligned, such as the normal Linux libc.so.
This trick helps avoid the problems associated with cross-compilation.
A lot of build systems expect to be able to run the executables they
build. By putting "ncrewrite --nop" into a gcc wrapper, the
executables built will be directly runnable.
This means that the Python build now runs to completion when building
with nacl-glibc; it no longer segfaults when trying to run the python
executable it has built outside of sel_ldr.
This is based on an idea I wrote down in a blog post earlier in the year:
http://lackingrhoticity.blogspot.com/2009/01/on-abi-and-api-compatibility.html
diff --git a/ncv/build.scons b/ncv/build.scons
index 413d53f..ddf980b 100644
--- a/ncv/build.scons
+++ b/ncv/build.scons
@@ -125,6 +125,8 @@ ncval = env.ComponentProgram('ncval', ['ncval.c', 'ncval_tests.c'])
env.Requires(ncval, crt)
env.Requires(ncval, sdl_dll)
+env.ComponentProgram('ncrewrite', 'ncrewrite.c')
+
# TODO: get OSX and Windows working below here
if env['TARGET_PLATFORM'] in ['WINDOWS', 'MAC']:
diff --git a/ncv/ncrewrite.c b/ncv/ncrewrite.c
new file mode 100644
index 0000000..83b83d4
--- /dev/null
+++ b/ncv/ncrewrite.c
@@ -0,0 +1,100 @@
+
+#include <assert.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "native_client/include/elf.h"
+#include "native_client/ncv/ncdecode.h"
+
+
+static int write_nops = 0;
+
+static void handle_instruction(const struct NCDecoderState *mstate)
+{
+ /* Rewrites
+ nop; nop; nop
+ jmp *%reg
+ to
+ and $0xffffffe0, %reg
+ jmp *%reg
+ */
+ if(mstate->opinfo->insttype == NACLi_INDIRECT) {
+ char and_instr[3] = {
+ 0x83, /* "and" opcode */
+ 0xe0 | modrm_rm(mstate->inst.maddr[1]), /* operand */
+ 0xe0, /* alignment mask, constant ~31 */
+ };
+
+ if((PreviousInst(mstate, -1) != NULL &&
+ memcmp(mstate->inst.maddr - 3, and_instr, 3) == 0) ||
+ (PreviousInst(mstate, -3) != NULL &&
+ mstate->inst.maddr[-3] == 0x90 &&
+ mstate->inst.maddr[-2] == 0x90 &&
+ mstate->inst.maddr[-1] == 0x90)) {
+ if(write_nops)
+ memset(mstate->inst.maddr - 3, 0x90, 3);
+ else
+ memcpy(mstate->inst.maddr - 3, and_instr, 3);
+ }
+ else {
+ fprintf(stderr, "%08x: cannot rewrite\n", mstate->inst.vaddr);
+ }
+ }
+}
+
+static void fixup_file(const char *filename)
+{
+ int fd = open(filename, O_RDWR);
+ if(fd < 0) {
+ perror("open");
+ exit(1);
+ }
+ struct stat st;
+ if(fstat(fd, &st) < 0) {
+ perror("fstat");
+ exit(1);
+ }
+ unsigned char *data = mmap(NULL, st.st_size, PROT_READ | PROT_WRITE,
+ MAP_SHARED, fd, 0);
+ if(data == MAP_FAILED) {
+ perror("mmap");
+ exit(1);
+ }
+ if(close(fd) < 0) {
+ perror("close");
+ exit(1);
+ }
+ Elf32_Ehdr *header = (Elf32_Ehdr *) data;
+ assert(memcmp(header->e_ident, ELFMAG, strlen(ELFMAG)) == 0);
+ int i;
+ for(i = 0; i < header->e_shnum; i++) {
+ Elf32_Shdr *section = (Elf32_Shdr *) (data + header->e_shoff +
+ header->e_shentsize * i);
+ if((section->sh_flags & SHF_EXECINSTR) != 0) {
+ NCDecodeSegment(data + section->sh_offset, section->sh_addr,
+ section->sh_size, NULL);
+ }
+ }
+ if(munmap(data, st.st_size) < 0) {
+ perror("munmap");
+ exit(1);
+ }
+}
+
+int main(int argc, const char *argv[])
+{
+ int i;
+ NCDecodeRegisterCallbacks(handle_instruction, NULL, NULL, NULL);
+ for(i = 1; i < argc; i++) {
+ const char *arg = argv[i];
+ if(strcmp(arg, "--nop") == 0)
+ write_nops = 1;
+ else
+ fixup_file(arg);
+ }
+ return 0;
+}
diff --git a/ncv/ncrewrite_test.py b/ncv/ncrewrite_test.py
new file mode 100644
index 0000000..69d80ff
--- /dev/null
+++ b/ncv/ncrewrite_test.py
@@ -0,0 +1,137 @@
+
+import os
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+
+def write_file(filename, data):
+ fh = open(filename, "w")
+ try:
+ fh.write(data)
+ finally:
+ fh.close()
+
+
+def read_file(filename):
+ fh = open(filename, "r")
+ try:
+ return fh.read()
+ finally:
+ fh.close()
+
+
+class TempDirTestCase(unittest.TestCase):
+
+ def setUp(self):
+ self._on_teardown = []
+
+ def make_temp_dir(self):
+ temp_dir = tempfile.mkdtemp(prefix="tmp-%s-" % self.__class__.__name__)
+ def tear_down():
+ shutil.rmtree(temp_dir)
+ self._on_teardown.append(tear_down)
+ return temp_dir
+
+ def tearDown(self):
+ for func in reversed(self._on_teardown):
+ func()
+
+
+class JumpRewriterTest(TempDirTestCase):
+
+ def _assemble(self, asm_source):
+ asm_file = os.path.join(self.make_temp_dir(), "foo.S")
+ obj_file = os.path.join(self.make_temp_dir(), "foo.o")
+ write_file(asm_file, asm_source)
+ subprocess.check_call(["gcc", "-c", asm_file, "-o", obj_file])
+ return obj_file
+
+ def _disassemble(self, obj_file):
+ proc = subprocess.Popen(["objdump", "-d", obj_file],
+ stdout=subprocess.PIPE)
+ return proc.communicate()[0]
+
+ def assert_object_files_equal(self, obj_file, obj_file_expect):
+ if read_file(obj_file) != read_file(obj_file_expect):
+ raise AssertionError("Unexpected output:\n%s\nExpected:\n%s" %
+ (self._disassemble(obj_file),
+ self._disassemble(obj_file_expect)))
+
+ def test_rewriting(self):
+ original = """
+ // Instructions to be rewritten
+ nop; nop; nop
+ jmp *%eax
+ nop; nop; nop
+ jmp *%ebx
+ nop; nop; nop
+ jmp *%ecx
+ nop; nop; nop
+ jmp *%edx
+
+ nop; nop; nop
+ call *%eax
+ nop; nop; nop
+ call *%ebx
+ nop; nop; nop
+ call *%ecx
+ nop; nop; nop
+ call *%edx
+"""
+ rewritten = """
+ and $0xffffffe0, %eax
+ jmp *%eax
+ and $0xffffffe0, %ebx
+ jmp *%ebx
+ and $0xffffffe0, %ecx
+ jmp *%ecx
+ and $0xffffffe0, %edx
+ jmp *%edx
+
+ and $0xffffffe0, %eax
+ call *%eax
+ and $0xffffffe0, %ebx
+ call *%ebx
+ and $0xffffffe0, %ecx
+ call *%ecx
+ and $0xffffffe0, %edx
+ call *%edx
+"""
+ leave_alone = """
+ // These should be left alone
+ nop; nop; nop
+ jmp 0x12345678
+ mov $123, %eax
+"""
+ obj_file = self._assemble(original + leave_alone)
+ obj_file_expect = self._assemble(rewritten + leave_alone)
+ subprocess.check_call(["ncrewrite", obj_file])
+ self.assert_object_files_equal(obj_file, obj_file_expect)
+
+ obj_file = self._assemble(rewritten + leave_alone)
+ obj_file_expect = self._assemble(original + leave_alone)
+ subprocess.check_call(["ncrewrite", "--nop", obj_file])
+ self.assert_object_files_equal(obj_file, obj_file_expect)
+
+ def test_rewriting_missing_nops(self):
+ input_data = """
+ // Not enough preceding nops to rewrite
+ nop; nop
+ jmp *%ecx
+"""
+ obj_file = self._assemble(input_data * 2)
+ original = read_file(obj_file)
+ proc = subprocess.Popen(["ncrewrite", obj_file], stderr=subprocess.PIPE)
+ stderr = proc.communicate()[1]
+ self.assertEquals(stderr,
+ "00000002: cannot rewrite\n"
+ "00000006: cannot rewrite\n")
+ self.assertEquals(proc.wait(), 0)
+ # Object file should not have been changed.
+ self.assertEquals(original, read_file(obj_file))
+
+
+if __name__ == "__main__":
+ unittest.main()