Opened 12 years ago

Closed 12 years ago

Last modified 12 years ago

#284 closed defect (wontfix)

Initial mmap fails on starting lx86cl64: "remap spjump: Permission denied"

Reported by: vii Owned by: gb
Priority: blocker Milestone:
Component: Runtime (threads, GC) Version:
Keywords: Cc:

Description (last modified by gb)

I am using Ubuntu hardy, Linux hayaji 2.6.24-16-generic #1 SMP Thu Apr 10 12:47:45 UTC 2008 x86_64 GNU/Linux.

Starting openmcl from the 070722 snapshot, or any other that I have tried, gives this error:

remap spjump: Permission denied

I strace'd the process and it is a mmap that fails

brk(0)                                  = 0x627000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f5819bb4000
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f5819bb2000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=88965, ...}) = 0
mmap(NULL, 88965, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f5819b9c000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/libdl.so.2", O_RDONLY)       = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 \16\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=14624, ...}) = 0
mmap(NULL, 2109728, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f5819794000
mprotect(0x7f5819796000, 2097152, PROT_NONE) = 0
mmap(0x7f5819996000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f5819996000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/libm.so.6", O_RDONLY)        = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260>\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=526560, ...}) = 0
mmap(NULL, 2621672, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f5819513000
mprotect(0x7f5819593000, 2093056, PROT_NONE) = 0
mmap(0x7f5819792000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7f000) = 0x7f5819792000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/libpthread.so.0", O_RDONLY)  = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260W\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=130224, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f5819b9b000
mmap(NULL, 2208624, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f58192f7000
mprotect(0x7f581930d000, 2097152, PROT_NONE) = 0
mmap(0x7f581950d000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f581950d000
mmap(0x7f581950f000, 13168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f581950f000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/libc.so.6", O_RDONLY)        = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\342"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1436976, ...}) = 0
mmap(NULL, 3543672, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f5818f95000
mprotect(0x7f58190ed000, 2097152, PROT_NONE) = 0
mmap(0x7f58192ed000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x158000) = 0x7f58192ed000
mmap(0x7f58192f2000, 17016, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f58192f2000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f5819b9a000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f5819b99000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f5819b98000
arch_prctl(ARCH_SET_FS, 0x7f5819b99860) = 0
mprotect(0x7f58192ed000, 12288, PROT_READ) = 0
munmap(0x7f5819b9c000, 88965)           = 0
set_tid_address(0x7f5819b998f0)         = 23429
set_robust_list(0x7f5819b99900, 0x18)   = 0
futex(0x7fff21bb5a4c, 0x81 /* FUTEX_??? */, 1) = 0
rt_sigaction(SIGRTMIN, {0x7f58192fc2d0, [], SA_RESTORER|SA_SIGINFO, 0x7f58193057d0}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7f58192fc350, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f58193057d0}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
uname({sys="Linux", node="hayaji", ...}) = 0
readlink("/proc/self/exe", "/tmp/ccl/lx86cl64", 4096) = 17
brk(0)                                  = 0x627000
brk(0x648000)                           = 0x648000
arch_prctl(ARCH_GET_GS, [0])            = 0
arch_prctl(ARCH_GET_FS, [0x7f5819b99860]) = 0
mmap(0x5000, 4096, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = -1 EACCES (Permission denied)
dup(2)                                  = 3
fcntl(3, F_GETFL)                       = 0x8002 (flags O_RDWR|O_LARGEFILE)
fstat(3, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 0), ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f5819bb1000
lseek(3, 0, SEEK_CUR)                   = -1 ESPIPE (Illegal seek)
write(3, "remap spjump: Permission denied\n", 32remap spjump: Permission denied
) = 32
close(3)                                = 0
munmap(0x7f5819bb1000, 4096)            = 0
exit_group(1)                           = ?

Attachments (1)

straced-start.txt (4.9 KB) - added by vii 12 years ago.
strace output with newlines hopefully in place

Download all attachments as: .zip

Change History (5)

Changed 12 years ago by vii

strace output with newlines hopefully in place

comment:1 Changed 12 years ago by gb

  • Description modified (diff)
  • Status changed from new to assigned

I haven't installed the Ubuntu 8.04 release candidate yet (8.04 isn't -quite- out yet); I'll try to do that in the next few days.

I did try a Fedora system that's running 2.6.24.4 and didn't notice any such problem, so it's not just the (approximate) kernel version. (There do seem to have been some changes in mmap() behavior in 2.6.24, apparently trying to close a rather large security hole; I suppose that this might be a garbled artifact of that ...)

But I really don't know, and won't have a good idea until I can look at it a bit.

comment:2 Changed 12 years ago by gb

  • Resolution set to wontfix
  • Status changed from assigned to closed

I was able to install the release candidate of Kubuntu 8.04 and reproduce this.

The bad news is that a very preliminary diagnosis suggests that it's a bug in the Linux kernel; the only documented way for mmap() to return a permission error (EPERM) involves cases where a non-anonymous mapping (a file mapping) is being created, PROT_EXEC access is requested, and the file is mounted on a filesystem that doesn't allow execute permissions. remap_spjump() passes the PROT_EXEC flag even though it really doesn't need to, but it's also trying to create a process-private, anonymous mapping. Even when PROT_EXEC is removed, we get the same EPERM error trying to map a page at #x5000 in our address space, and we really, really shouldn't; this does really seem like a botched attempt to patch a security hole.

The good news (relatively) is that CCL 1.2 tries to map a page 64K higher in the address space (at #x15000 instead of #x5000) for totally unrelated reasons, and that seems to work fine.

I have no idea whether Ubuntu's 2.6.24-16 kernel is newer or older than Fedora's 2.6.24.4-64; I don't see this problem under Fedora, but it's possible that I either will soon or would have recently, depending on how those numbering schemes relate to each other.

If I'm correct in assuming that it's a Linux kernel bug, there isn't much that I can do about it.

The best workaround at this point would seem to be for us to get CCL 1.2 released, since 1.2 doesn't seem to be affected.

comment:3 Changed 12 years ago by gb

Right after I hit 'submit changes' on the previous reply, I noticed a reference to 'mmap_min_addr' in the Debian Changelog for the kernel, and eventually this led to

shell> cat /proc/sys/vm/mmap_min_addr
65536

So, Ubuntu 8.04 is disallowing mmap() of addresses below 64K by default; CCL 1.1 wants to use some addresses below 64K and attempts to use such addresses fail, where CCL 1.2 happens to want to use some addresses slightly above the 64K limit and that works fine.

There are other problems with 1.1 and kernels >= 2.6.23; one workaround is to boot with the vdso=0 kernel option.

With that kernel option set and with /proc/sys/vm/mmap_min_addr set to 4K, we get:

[src/ccl] gb@shantaram> dmesg | fgrep vdso
[    0.000000] Command line: root=UUID=f4360cb9-5b0c-4a41-a7f7-0f8bb02a8ff0 ro quiet splash noapic vdso=0
[    0.000000] Kernel command line: root=UUID=f4360cb9-5b0c-4a41-a7f7-0f8bb02a8ff0 ro quiet splash noapic vdso=0
[src/ccl] gb@shantaram> cat /proc/sys/vm/mmap_min_addr
4096
[src/ccl] gb@shantaram> openmcl64
Welcome to Clozure Common Lisp Version 1.1-r9027S (LinuxX8664)!
?

comment:4 Changed 12 years ago by vii

Thanks for the prompt response and resolution.

Just in case someone else gets this: it will successfully run if you are root, even if the mmap_min_addr is high.

Note: See TracTickets for help on using tickets.