mmap of /dev/mem fails with invalid argument for virt_to_phys address, but address is page aligned

Question

For some reason my mmap failed with an Invalid argument message even though my offset is page aligned. Page size is 4096 bytes. Also CONFIG_STRICT_DEVMEM is disabled, i.e. I can access memory above 1MB.

Here is my code:

void *mmap64;
off_t offset = 0x000000d9fcc000;
int memFd = open("/dev/mem", O_RDWR);
if (-1 == memFd)
  perror("Error ");

mmap64 = mmap(0, getpagesize(), PROT_WRITE | PROT_READ, MAP_SHARED, memFd, offset);
if (MAP_FAILED == mmap64) {
  perror("Error ");
  return -1;
}

Can someone explain why this is happening?

EDIT

Here is the strace of my code

execve("./to_phys_test", ["./to_phys_test", "-r"], [/* 18 vars */]) = 0
brk(0)                                  = 0x2012000
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fe240a2c000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=162063, ...}) = 0
mmap(NULL, 162063, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fe240a04000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P \2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1840928, ...}) = 0
mmap(NULL, 3949248, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fe240447000
mprotect(0x7fe240601000, 2097152, PROT_NONE) = 0
mmap(0x7fe240801000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1ba000) = 0x7fe240801000
mmap(0x7fe240807000, 17088, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fe240807000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fe240a03000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fe240a01000
arch_prctl(ARCH_SET_FS, 0x7fe240a01740) = 0
mprotect(0x7fe240801000, 16384, PROT_READ) = 0
mprotect(0x601000, 4096, PROT_READ)     = 0
mprotect(0x7fe240a2e000, 4096, PROT_READ) = 0
munmap(0x7fe240a04000, 162063)          = 0
open("/dev/mem", O_RDWR)                = 3
open("/dev/my_kmodule", O_RDWR)    = 4
ioctl(4, 0x40086e00, 0x7ffc72b334b0)    = 0
fstat(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 0), ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fe240a2b000
write(1, "sa.size = 44\n", 13)          = 13
write(1, "sa.addr_uint64_t = d9047000\n", 28) = 28
write(1, "sa.addr_void_ptr = 0xd9047000\n", 30) = 30
write(1, "PAGE_SIZE = 4096\n", 17)      = 17
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_SHARED, 3, 0xd9047000) = -1 EINVAL (Invalid argument)
dup(2)                                  = 5
fcntl(5, F_GETFL)                       = 0x8002 (flags O_RDWR|O_LARGEFILE)
brk(0)                                  = 0x2012000
brk(0x2033000)                          = 0x2033000
fstat(5, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 0), ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fe240a2a000
lseek(5, 0, SEEK_CUR)                   = -1 ESPIPE (Illegal seek)
write(5, "Error : Invalid argument\n", 25) = 25
close(5)                                = 0
munmap(0x7fe240a2a000, 4096)            = 0
exit_group(-1)                          = ?
+++ exited with 255 +++

Answer 1

nopat kernel command line argument

Just add that and it works, as mentioned at: https://stackoverflow.com/a/36634422/895245

Here is my test setup:

kernel module:

#include <asm/io.h> /* virt_to_phys */
#include <linux/debugfs.h>
#include <linux/delay.h> /* usleep_range */
#include <linux/kernel.h>
#include <linux/kthread.h>
#include <linux/module.h>
#include <linux/seq_file.h> /* single_open, single_release */
#include <linux/slab.h> /* kmalloc, kfree */

static volatile u32 *i;

static struct dentry *debugfs_file;

static int show(struct seq_file *m, void *v)
{
    seq_printf(m,
        "*i 0x%llx\n"
        "i %p\n"
        "virt_to_phys 0x%llx\n",
        (unsigned long long)*i,
        i,
        (unsigned long long)virt_to_phys((void *)i)
    );
    return 0;
}

static int open(struct inode *inode, struct  file *file)
{
    return single_open(file, show, NULL);
}

static const struct file_operations fops = {
    .llseek = seq_lseek,
    .open = open,
    .owner = THIS_MODULE,
    .read = seq_read,
    .release = single_release,
};

static int myinit(void)
{
    i = kmalloc(sizeof(i), GFP_KERNEL);
    *i = 0x12345678;
    debugfs_file = debugfs_create_file(
        "lkmc_virt_to_phys", S_IRUSR, NULL, NULL, &fops);
    return 0;
}

static void myexit(void)
{
    debugfs_remove(debugfs_file);
    kfree((void *)i);
}

module_init(myinit)
module_exit(myexit)
MODULE_LICENSE("GPL");

/dev/mem userland:

#!/bin/sh
set -ex
insmod /virt_to_phys.ko
cd /sys/kernel/debug
cat lkmc_virt_to_phys
# *i = 0x12345678
addr=$(grep virt_to_phys lkmc_virt_to_phys | cut -d ' ' -f 2)
devmem2 "$addr"
devmem2 "$addr" w 0x9ABCDEF0
cat lkmc_virt_to_phys
# *i = 0x9ABCDEF0
rmmod virt_to_phys

nopat being passed at: https://github.com/cirosantilli/linux-kernel-module-cheat/blob/2eca9280e12dbab79ccb67d0640b2a0edc2c9ffc/runqemu#L65

Also try xp on QEMU monitor.

And devmem2 is upstreamed by Buildroot itself: http://free-electrons.com/pub/mirror/devmem2.c See also: Accessing physical address from user space