maybe dumb question about RCU

[Jeff Haran]

Hi all,

I've written some sample code to test out the question I raised earlier and it seems that if you call rcu_dereference() on a given RCU protected pointer more than once within a read critical section delimited by rcu_read_lock() and rcu_read_unlock(), you can get different values. Maybe this is well known to experienced kernel developers, but it was unclear to me from the documentation I'd read so I figured I'd share what I found with my fellow newbies.

The code for the sample kernel module is copied at the end if you'd like to try it yourself. Just insmod the resulting rcutest.ko. When I do so, I get this on my console:

[root at s01b01 ~]# [  496.970678] rcutest initialized
[  499.968538] rcu_reader_thread a = 0xffffffffa00ab474 b = 0xffffffffa00ab475 off_count = 1 count = 3
[  503.975844] rcu_reader_thread a = 0xffffffffa00ab475 b = 0xffffffffa00ab474 off_count = 2 count = 7
[  507.983410] rcu_reader_thread a = 0xffffffffa00ab474 b = 0xffffffffa00ab475 off_count = 3 count = 11
[  511.990554] rcu_reader_thread a = 0xffffffffa00ab475 b = 0xffffffffa00ab474 off_count = 4 count = 15
[  515.997486] rcu_reader_thread a = 0xffffffffa00ab474 b = 0xffffffffa00ab475 off_count = 5 count = 19
[  520.005163] rcu_reader_thread a = 0xffffffffa00ab475 b = 0xffffffffa00ab474 off_count = 6 count = 23
[  524.012020] rcu_reader_thread a = 0xffffffffa00ab474 b = 0xffffffffa00ab475 off_count = 7 count = 27
[  528.019520] rcu_reader_thread a = 0xffffffffa00ab475 b = 0xffffffffa00ab474 off_count = 8 count = 31
[  532.026716] rcu_reader_thread a = 0xffffffffa00ab474 b = 0xffffffffa00ab475 off_count = 9 count = 35
[  536.034235] rcu_reader_thread a = 0xffffffffa00ab475 b = 0xffffffffa00ab474 off_count = 10 count = 39
[  540.041010] rcu_reader_thread a = 0xffffffffa00ab474 b = 0xffffffffa00ab475 off_count = 11 count = 43
[  544.048265] rcu_reader_thread a = 0xffffffffa00ab475 b = 0xffffffffa00ab474 off_count = 12 count = 47
...

Indicating that the values obtained within the critical section can be different. So it would seem that the best practice when reading these RCU protected pointers is to get them once and only once within the critical section via a single call to rcu_dereference_pointer(), store that pointer someplace local and then operate on the copy rather than make multiple calls to rcu_dereference_pointer().

The files copied below are the source rcutest.c, a Makefile and a Kbuild. You might have to fiddle with the Makefile to get it to work on your system. We build against many kernel versions here so the Makefile is setup to point to different kernel versions.

Jeff Haran

$ cat rcutest.c
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/version.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/kdev_t.h>
#include <linux/fs.h>
#include <linux/device.h>
#include <linux/cdev.h>
#include <linux/uaccess.h>
#include <linux/sched.h>
#include <linux/wait.h>
#include <linux/list.h>
#include <linux/kthread.h>

char rcu_test_1[1];
char rcu_test_2[1];
char *rcu_pointer = rcu_test_1;
struct task_struct *updater_task;
struct task_struct *reader_task;
int off_count = 0;

static int rcu_updater_thread(void *arg)
{
        int count = 0;

        while (!kthread_should_stop()) {
                if (count & 1) {
                        rcu_assign_pointer(rcu_pointer, rcu_test_2);
                } else {
                        rcu_assign_pointer(rcu_pointer, rcu_test_1);
                }

                synchronize_rcu();
                count++;
                msleep(1500);
        }

        return 0;
}

static int rcu_reader_thread(void *arg)
{
        int count = 0;

        while (!kthread_should_stop()) {
                char *a;
                char *b;
                u64 start;

                count++;
                rcu_read_lock();
                a = rcu_dereference(rcu_pointer);
                start = get_jiffies_64();
                while (get_jiffies_64() < (start + 1000));
                b = rcu_dereference(rcu_pointer);
                rcu_read_unlock();
                if (a != b) {
                        off_count++;
                        printk(KERN_INFO "rcu_reader_thread a = 0x%p b = 0x%p off_count = %d count = %d\n",
                                a, b, off_count, count);
                }
                if (need_resched()) {
                        schedule();
                }
        }

        return 0;
}

static int startrcuupdater(void)
{
        int rc = 0;

        updater_task = kthread_create(rcu_updater_thread, 0, "rcuupdater");
        if (IS_ERR(updater_task)) {
                rc = PTR_ERR(updater_task);
                printk(KERN_ERR "rcu_updater create failed %d\n", rc);
                goto err;
        }

        wake_up_process(updater_task);

err:
        return rc;
}

static int startrcureader(void)
{
        int rc = 0;

        reader_task = kthread_create(rcu_reader_thread, 0, "rcureader");
        if (IS_ERR(reader_task)) {
                rc = PTR_ERR(reader_task);
                printk(KERN_ERR "rcu_updater create failed %d\n", rc);
                goto err;
        }

        wake_up_process(reader_task);

err:
        return rc;
}

static int __init rcu_test_init(void)
{
        int rc;

        rc = startrcuupdater();
        if (rc) {
                goto err;
        }
        rc = startrcureader();
        if (rc) {
                kthread_stop(updater_task);
                goto err;
        }
        printk(KERN_INFO "rcutest initialized\n");

err:

        return rc;
}

static void __exit rcu_test_exit(void)
{
        kthread_stop(reader_task);
        kthread_stop(updater_task);
    printk(KERN_INFO "rcutest exited\n");
}

module_init(rcu_test_init);
module_exit(rcu_test_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Jeff Haran");
MODULE_DESCRIPTION("a test of RCU");

$ cat Makefile

SANDBOX:=/s/fusion/33

#UNAME-R := $(subst ${SANDBOX}/lib/modules/,,${VER})
UNAME-R:=2.6.32-504.8.1.el6.x86_64

all:
        make -C ${SANDBOX}/lib/modules/${UNAME-R}/build M=$(CURDIR) RCUTEST_BASE=${CURDIR} modules

clean:
        make -C ${SANDBOX}/lib/modules/${UNAME-R}/build M=$(CURDIR) clean
        rm -f Module.markers Module.symvers

$ cat Kbuild
ccflags-y += -I$(RCUTEST_BASE)/include -Werror

obj-m += rcutest.o