x86: Add support for SW-switched segment-based protection domains

This patch extends the protection domain framework with a third plugin
that is a hybrid of the previous two.  The hardware task switching
mechanism has a strictly-defined format for TSS data structures that
causes more space to be consumed than would otherwise be required.
This patch defines a smaller data structure that is allocated for each
protection domain, only requiring 32 bytes instead of 128 bytes.  It
uses the same multi-segment memory layout as the TSS-based plugin and
leaves paging disabled.  However, it uses a similar mechanism as the
paging plugin to perform system call dispatches and returns.

For additional information, please refer to cpu/x86/mm/README.md.

cpu/x86/Makefile.x86_quarkX1000

@@ -25,6 +25,11 @@ else ifeq ($(X86_CONF_PROT_DOMAINS),tss)
 CFLAGS += -DX86_CONF_PROT_DOMAINS=2
 X86_CONF_MULTI_SEG = 1
 CONTIKI_SOURCEFILES += tss-prot-domains-asm.S
+else ifeq ($(X86_CONF_PROT_DOMAINS),swseg)
+# This matches the definition of X86_CONF_PROT_DOMAINS__SWSEG in prot-domains.h:
+CFLAGS += -DX86_CONF_PROT_DOMAINS=3
+X86_CONF_SYSCALLS_INT = 1
+X86_CONF_MULTI_SEG = 1
 else
 $(error Unrecognized setting for X86_CONF_PROT_DOMAINS: \
         $(X86_CONF_PROT_DOMAINS).  See cpu/x86/mm/README.md for \

cpu/x86/mm/README.md

@@ -6,11 +6,12 @@ Introduction
 
 The X86 port of Contiki implements a simple, lightweight form of
 protection domains using a pluggable framework.  Currently, there are
-two plugins available:
+three plugins available:
 
  - Flat memory model with paging.
- - Multi-segment memory model with hardware-switched segments based on
+ - Multi-segment memory model with either hardware- or
-   Task-State Segment (TSS) structures.
+   software-switched segments.  The hardware-switched segments
+   approach is based on Task-State Segment (TSS) structures.
 
 For an introduction to paging and TSS and possible ways in which they
 can be used, refer to the following resources:
@@ -144,8 +145,8 @@ Similarly, register contents may be accessed and modified across
 protection domain boundaries in some protection domain
 implementations.  The TSS task switching mechanism automatically saves
 and restores many registers to and from TSS data structures when
-switching tasks, but the paging-based protection domain implementation
+switching tasks, but the other protection domain implementations do
-does not perform analogous operations.
+not perform analogous operations.
 
 For the reasons described above, each protection domain should only
 invoke other protection domains that it trusts to properly handle data
@@ -847,6 +848,25 @@ in an unexpected manner, since segment register load instructions are
 unprivileged.  Similar segment register updates must be performed for
 similar reasons when dispatching system calls.
 
+### Software-Switched Segment-Based Protection Domains
+
+Primary implementation sources:
+
+ - cpu/x86/mm/swseg-prot-domains.c
+
+The requirement to allocate a TSS for each protection domain in the
+hardware-switched segments plugin may consume a substantial amount of
+space, since the size of each TSS is fixed by hardware to be at least
+104 bytes.  The software-switched segments plugin saves space by
+defining a more compact PDCS.  However, the layout and definitions of
+the segments is identical to what was described above for the
+hardware-switched segments plugin.
+
+The system call and return procedure is mostly identical to that for
+paging-based protection domains.  However, instead of updating and
+invalidating page tables, the dispatchers update the LDT and some of
+the segment registers.
+
 ### Pointer Validation
 
 Primary implementation sources:
@@ -957,6 +977,7 @@ the command line and specify one of the following options:
 
  - paging
  - tss
+ - swseg
 
 The paging option accepts a sub-option to determine whether the TLB is
 fully- or selectively-invalidated during protection domain switches.

cpu/x86/mm/gdt-layout.h

@@ -92,8 +92,13 @@
 /** Stack segment for exception handlers */
 #define GDT_IDX_STK_EXC       10
 
+#if X86_CONF_PROT_DOMAINS == X86_CONF_PROT_DOMAINS__TSS
 #define GDT_IDX_TSS(dom_id)   (GDT_NUM_FIXED_DESC + (2 * (dom_id)))
 #define GDT_IDX_LDT(dom_id)   (GDT_NUM_FIXED_DESC + (2 * (dom_id)) + 1)
+#else
+#define GDT_IDX_LDT(dom_id)   (GDT_NUM_FIXED_DESC + (dom_id))
+#endif
+
 #endif
 #else
 #define GDT_IDX_CODE          GDT_IDX_CODE_FLAT

cpu/x86/mm/multi-segment.c

@@ -139,7 +139,14 @@ prot_domains_gdt_init()
                     (uint32_t)&_stext_addr,
                     ((uint32_t)&_etext_addr) - (uint32_t)&_stext_addr,
                     SEG_FLAG(DPL, PRIV_LVL_EXC) | SEG_GRAN_BYTE |
-                    SEG_DESCTYPE_NSYS | SEG_TYPE_CODE_EX);
+                    SEG_DESCTYPE_NSYS |
+#if X86_CONF_PROT_DOMAINS == X86_CONF_PROT_DOMAINS__SWSEG
+  /* The general protection fault handler requires read access to CS */
+                    SEG_TYPE_CODE_EXRD
+#else
+                    SEG_TYPE_CODE_EX
+#endif
+                    );
   gdt_insert_boot(GDT_IDX_CODE_EXC, desc);
 
   segment_desc_init(&desc,
@@ -180,7 +187,9 @@ prot_domains_gdt_init()
    */
   desc.raw = SEG_DESC_NOT_PRESENT;
   for(i = 0; i < PROT_DOMAINS_ACTUAL_CNT; i++) {
+#if X86_CONF_PROT_DOMAINS == X86_CONF_PROT_DOMAINS__TSS
     gdt_insert_boot(GDT_IDX_TSS(i), desc);
+#endif
     gdt_insert_boot(GDT_IDX_LDT(i), desc);
   }
 

cpu/x86/mm/prot-domains.c

@@ -54,6 +54,13 @@ prot_domains_init(void)
   segment_desc_t desc;
 
   gdt_lookup(GDT_IDX_CODE_EXC, &desc);
+#if X86_CONF_PROT_DOMAINS == X86_CONF_PROT_DOMAINS__SWSEG
+  /* The exception code segment needs to be readable so that the general
+   * protection fault handler can decode instructions, but the interrupt and
+   * user level code segments should not be.
+   */
+  SEG_SET_FLAG(desc, TYPE, SEG_TYPE_CODE_EX);
+#endif
 
   SEG_SET_FLAG(desc, DPL, PRIV_LVL_INT);
   gdt_insert(GDT_IDX_CODE_INT, desc);

cpu/x86/mm/prot-domains.h

@@ -41,9 +41,11 @@
 #define X86_CONF_PROT_DOMAINS__NONE   0
 #define X86_CONF_PROT_DOMAINS__PAGING 1
 #define X86_CONF_PROT_DOMAINS__TSS    2
+#define X86_CONF_PROT_DOMAINS__SWSEG  3
 
 #define X86_CONF_PROT_DOMAINS_MULTI_SEG                                       \
-  (X86_CONF_PROT_DOMAINS == X86_CONF_PROT_DOMAINS__TSS)
+  ((X86_CONF_PROT_DOMAINS == X86_CONF_PROT_DOMAINS__TSS) ||                    \
+   (X86_CONF_PROT_DOMAINS == X86_CONF_PROT_DOMAINS__SWSEG))
 
 /** Privilege level (ring) for exception handlers and other supervisory code */
 #define PRIV_LVL_EXC  0
@@ -74,6 +76,8 @@ typedef uint32_t dom_id_t;
 #include "paging-prot-domains.h"
 #elif X86_CONF_PROT_DOMAINS == X86_CONF_PROT_DOMAINS__TSS
 #include "tss-prot-domains.h"
+#elif X86_CONF_PROT_DOMAINS == X86_CONF_PROT_DOMAINS__SWSEG
+#include "swseg-prot-domains.h"
 #endif
 
 #ifndef ATTR_META_ADDR_SPACE

cpu/x86/mm/stacks.h

@@ -61,6 +61,12 @@
 #else
 #define STACKS_SIZE_EXC 256
 #endif
+#elif X86_CONF_PROT_DOMAINS == X86_CONF_PROT_DOMAINS__SWSEG
+#ifdef __clang__
+#define STACKS_SIZE_EXC 512
+#else
+#define STACKS_SIZE_EXC 256
+#endif
 #elif X86_CONF_PROT_DOMAINS == X86_CONF_PROT_DOMAINS__TSS
 /**
  * This should be large enough to execute the exception handler with the

cpu/x86/mm/swseg-prot-domains.c

@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2015, Intel Corporation. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "gdt.h"
+#include "helpers.h"
+#include "multi-segment.h"
+#include "prot-domains.h"
+
+/*---------------------------------------------------------------------------*/
+void
+prot_domains_reg(dom_client_data_t ATTR_KERN_ADDR_SPACE *dcd,
+                 uintptr_t mmio, size_t mmio_sz,
+                 uintptr_t meta, size_t meta_sz,
+                 bool pio)
+{
+  volatile dom_kern_data_t ATTR_KERN_ADDR_SPACE *dkd;
+  dom_id_t dom_id;
+
+  KERN_READL(dom_id, dcd->dom_id);
+
+  if(PROT_DOMAINS_ACTUAL_CNT <= dom_id) {
+    halt();
+  }
+
+  dkd = prot_domains_kern_data + dom_id;
+
+  prot_domains_reg_multi_seg(dkd, mmio, mmio_sz, meta, meta_sz);
+
+  KERN_WRITEL(dkd->flags, pio ? PROT_DOMAINS_FLAG_PIO : 0);
+}
+/*---------------------------------------------------------------------------*/
+static inline void __attribute__((always_inline))
+prot_domains_switch(dom_id_t from_id, dom_id_t to_id,
+                    interrupt_stack_t *intr_stk)
+{
+  __asm__ __volatile__ (
+    "lldt %[_ldt_]\n\t"
+    "mov %[_meta_seg_], %%eax\n\t"
+    "lsl %%eax, %%ecx\n\t"
+    "jz 1f\n\t" /* ZF will only be set if the segment descriptor is valid. */
+    "xor %%eax, %%eax\n\t" /* Nullify metadata selector */
+    "1: mov %%eax, %%" SEG_META "s\n\t"
+    "mov %[_kern_seg_], %%eax\n\t"
+    "mov %%eax, %%" SEG_KERN "s\n\t"
+    :
+    : [_ldt_] "r" ((uint16_t)GDT_SEL_LDT(to_id)),
+      [_meta_seg_] "i" (LDT_SEL_META),
+      [_kern_seg_] "i" (LDT_SEL_KERN)
+    : "cc", "eax", "ecx"
+    );
+}
+/*---------------------------------------------------------------------------*/
+
+/* Enable inter-procedural optimization with procedures in the following file:
+ */
+#include "syscalls-int.c"

cpu/x86/mm/swseg-prot-domains.h

@@ -0,0 +1,86 @@
+/*
+ * Copyright (C) 2015, Intel Corporation. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef CPU_X86_MM_SWSEG_PROT_DOMAINS_H_
+#define CPU_X86_MM_SWSEG_PROT_DOMAINS_H_
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include "ldt-layout.h"
+#include "paging.h"
+#include "segmentation.h"
+#include "syscalls-int.h"
+
+struct dom_kern_data {
+  /** Local Descriptor Table with per-domain descriptors */
+  segment_desc_t ldt[LDT_NUM_DESC];
+  /** Flags are defined with the prefix PROT_DOMAINS_FLAG in prot-domains.h */
+  uint32_t flags;
+  /**
+   * Original return address from call stack when this protection domain
+   * invoked some other protection domain.  This serves to control the return
+   * entrypoint.  The callee is not permitted to modify this value (unless the
+   * callee is the kernel protection domain).
+   */
+  uintptr_t orig_ret_addr;
+
+  /* This structure is precisely 32 bytes in length, a power of 2.  If its size
+   * changes, add an alignment attribute to keep it aligned at a power of 2 so
+   * that dereferencing arrays of these structures uses shift instructions
+   * instead of multiplication.  Shifting is faster than multiplication.
+   */
+};
+
+/* relies on dom_kern_data: */
+#include "multi-segment.h"
+
+#define PROT_DOMAINS_ENTER_ISR(exc)                                           \
+  MULTI_SEGMENT_ENTER_ISR(exc)                                                \
+  PROT_DOMAINS_ENTER_ISR_COMMON(exc)
+#define PROT_DOMAINS_LEAVE_ISR(exc)                                           \
+  PROT_DOMAINS_LEAVE_ISR_COMMON(exc)                                          \
+  MULTI_SEGMENT_LEAVE_ISR(exc)
+
+#define prot_domains_impl_init syscalls_int_init
+
+#define prot_domains_set_wp(en)
+
+/* Allocate one additional GDT entry for each protection domain.  Note that
+ * the particular storage allocated by this statement may actually be used for
+ * some other protection domain, depending on how the linker happens to arrange
+ * all of the GDT storage.  The GDT_IDX_LDT macro in gdt-layout.h determine
+ * which storage is used for each protection domain.  Thus, this storage should
+ * not be referenced directly by its variable name.
+ */
+#define PROT_DOMAINS_ALLOC_IMPL(nm)                                           \
+  static segment_desc_t ATTR_BSS_GDT_MID _gdt_storage_##nm
+
+#endif /* CPU_X86_MM_SWSEG_PROT_DOMAINS_H_ */

cpu/x86/mm/syscalls-int-asm.S

@@ -33,6 +33,10 @@
 #include "gdt-layout.h"
 #include "stacks.h"
 
+/* Must match definitions (plus the trailing 's') in multi-segment.h */
+#define SEG_MMIO fs
+#define SEG_KERN fs
+
 .text
 
 /* Invoke the system call return dispatcher from the default privilege
@@ -42,21 +46,57 @@
 prot_domains_sysret_stub:
   int $PROT_DOMAINS_SYSRET_DISPATCH_INT
 
+.macro save_segs
+#if X86_CONF_PROT_DOMAINS == X86_CONF_PROT_DOMAINS__SWSEG
+  /* Save (and restore, in restore_segs) MMIO segment register into
+   * callee-saved register in case a system call was invoked from a region in
+   * which MMIO is enabled.
+   */
+  push %SEG_MMIO
+#endif
+.endm
+
+.macro restore_segs
+#if X86_CONF_PROT_DOMAINS == X86_CONF_PROT_DOMAINS__SWSEG
+  pop %SEG_MMIO
+#endif
+.endm
+
+/* Refresh most of the segment registers in case they were corrupted by
+ * userspace code to prevent that from corrupting the operation of the
+ * privileged code.
+ */
+.macro load_kern_segs
+#if X86_CONF_PROT_DOMAINS == X86_CONF_PROT_DOMAINS__SWSEG
+  mov $GDT_SEL_DATA, %eax
+  mov %eax, %ds
+  mov %eax, %es
+  mov $GDT_SEL_DATA_KERN_EXC, %eax
+  mov %eax, %SEG_KERN
+#endif
+.endm
+
 /* Invoke the system call dispatcher C routine */
 .global prot_domains_syscall_dispatcher
 prot_domains_syscall_dispatcher:
   mov %esp, %ecx /*< interrupt_stack_t *intr_stk */
   /* EDX already set to "dom_client_data_t to_dcd" by syscall stub */
-  push %eax /*< syscalls_id_t syscall_id */
+  save_segs
+  push %eax /*< syscalls_entrypoint_t *syscall */
+  load_kern_segs
   call prot_domains_syscall_dispatcher_impl
   /* fastcall convention, so callee pops arguments */
+  restore_segs
   iret
 
 /* Invoke the system call return dispatcher C routine */
 .global prot_domains_sysret_dispatcher
 prot_domains_sysret_dispatcher:
   mov %esp, %ecx /*< interrupt_stack_t *intr_stk */
+  save_segs
+  load_kern_segs
   call prot_domains_sysret_dispatcher_impl
+  restore_segs
   /* Zero caller-saved registers in case they contain secrets.  The system call
    * handlers and dispatchers need to preserve the callee-saved registers.
    */
@@ -67,11 +107,17 @@ prot_domains_sysret_dispatcher:
 
 .global prot_domains_launch_kernel
 prot_domains_launch_kernel:
+#if X86_CONF_PROT_DOMAINS == X86_CONF_PROT_DOMAINS__PAGING
   mov $GDT_SEL_DATA, %eax
   mov %eax, %ds
   mov %eax, %es
   mov %eax, %fs
   mov %eax, %gs
+#else
+  mov $GDT_SEL_LDT(DOM_ID_kern), %eax
+  lldt %ax
+  call multi_segment_launch_kernel
+#endif
   /* init interrupt return stack: */
   pushl $GDT_SEL_STK
   lea stacks_main, %eax

cpu/x86/mm/syscalls-int.c

@@ -91,7 +91,10 @@ syscall_dispatcher_tail(interrupt_stack_t *intr_stk,
                         uint32_t syscall_eip)
 {
   dom_id_t from_id;
-  volatile dom_kern_data_t *from_dkd, *to_dkd;
+  uint32_t tmp;
+  volatile dom_kern_data_t ATTR_KERN_ADDR_SPACE *from_dkd, *to_dkd;
+
+  uint32_t loc_call_stk_ptr;
 
   to_dkd = prot_domains_kern_data + to_id;
 
@@ -101,36 +104,40 @@ syscall_dispatcher_tail(interrupt_stack_t *intr_stk,
    * kernel data associated with that protection domain.  That model does not
    * permit reentrancy.
    */
-  if((to_dkd->flags & PROT_DOMAINS_FLAG_BUSY) == PROT_DOMAINS_FLAG_BUSY) {
+  KERN_READL(tmp, to_dkd->flags);
+  if((tmp & PROT_DOMAINS_FLAG_BUSY) == PROT_DOMAINS_FLAG_BUSY) {
     halt();
   }
-  to_dkd->flags |= PROT_DOMAINS_FLAG_BUSY;
+  tmp |= PROT_DOMAINS_FLAG_BUSY;
+  KERN_WRITEL(to_dkd->flags, tmp);
 
   /* Update the interrupt stack so that the IRET instruction will return to the
    * system call entrypoint.
    */
   intr_stk->eip = syscall_eip;
 
+  KERN_READL(loc_call_stk_ptr, inter_dom_call_stk_ptr);
   /* Lookup the information for the caller */
-  from_id = inter_dom_call_stk[inter_dom_call_stk_ptr - 1];
+  KERN_READL(from_id, inter_dom_call_stk[loc_call_stk_ptr - 1]);
   from_dkd = prot_domains_kern_data + from_id;
 
   /* Save the current return address from the unprivileged stack to a protected
    * location in the kernel-owned data structure.  This enforces return
    * entrypoint control.
    */
-  from_dkd->orig_ret_addr = *(uintptr_t *)intr_stk->esp;
+  KERN_WRITEL(from_dkd->orig_ret_addr, *(uintptr_t *)intr_stk->esp);
   /* Update the unprivileged stack so that when the system call body is
    * complete, it will invoke the system call return stub.
    */
   *((uintptr_t *)intr_stk->esp) = (uintptr_t)prot_domains_sysret_stub;
 
-  if(MAX_INTER_DOM_CALL_STK_SZ <= inter_dom_call_stk_ptr) {
+  if(MAX_INTER_DOM_CALL_STK_SZ <= loc_call_stk_ptr) {
     halt();
   }
-  inter_dom_call_stk[inter_dom_call_stk_ptr] = to_id;
+  KERN_WRITEL(inter_dom_call_stk[loc_call_stk_ptr], to_id);
 
-  inter_dom_call_stk_ptr++;
+  loc_call_stk_ptr++;
+  KERN_WRITEL(inter_dom_call_stk_ptr, loc_call_stk_ptr);
 
   dispatcher_tail(from_id, to_id, intr_stk);
 }
@@ -140,6 +147,7 @@ prot_domains_syscall_dispatcher_impl(interrupt_stack_t *intr_stk,
                                      dom_id_t to_id,
                                      syscalls_entrypoint_t *syscall)
 {
+  uint32_t tmp;
   uint32_t syscall_eip;
 
   if(PROT_DOMAINS_ACTUAL_CNT <= to_id) {
@@ -156,11 +164,12 @@ prot_domains_syscall_dispatcher_impl(interrupt_stack_t *intr_stk,
     halt();
   }
 
-  if((BIT(to_id) & syscall->doms) == 0) {
+  KERN_READL(tmp, syscall->doms);
+  if((BIT(to_id) & tmp) == 0) {
     halt();
   }
 
-  syscall_eip = syscall->entrypoint;
+  KERN_READL(syscall_eip, syscall->entrypoint);
 
   prot_domains_set_wp(false);
 
@@ -171,9 +180,9 @@ int main(void);
 void __attribute__((fastcall))
 prot_domains_launch_kernel_impl(interrupt_stack_t *intr_stk)
 {
-  inter_dom_call_stk[0] = DOM_ID_app;
+  KERN_WRITEL(inter_dom_call_stk[0], DOM_ID_app);
 
-  inter_dom_call_stk_ptr = 1;
+  KERN_WRITEL(inter_dom_call_stk_ptr, 1);
 
   syscall_dispatcher_tail(intr_stk, DOM_ID_kern, (uint32_t)main);
 }
@@ -182,20 +191,27 @@ void __attribute__((fastcall))
 prot_domains_sysret_dispatcher_impl(interrupt_stack_t *intr_stk)
 {
   dom_id_t from_id, to_id;
-  if(inter_dom_call_stk_ptr <= 1) {
+  uint32_t loc_call_stk_ptr;
+  uint32_t flags;
+
+  KERN_READL(loc_call_stk_ptr, inter_dom_call_stk_ptr);
+  if(loc_call_stk_ptr <= 1) {
     halt();
   }
 
-  from_id = inter_dom_call_stk[inter_dom_call_stk_ptr - 1];
+  KERN_READL(from_id, inter_dom_call_stk[loc_call_stk_ptr - 1]);
-  to_id = inter_dom_call_stk[inter_dom_call_stk_ptr - 2];
+  KERN_READL(to_id, inter_dom_call_stk[loc_call_stk_ptr - 2]);
 
-  intr_stk->eip = prot_domains_kern_data[to_id].orig_ret_addr;
+  KERN_READL(intr_stk->eip,
+            prot_domains_kern_data[to_id].orig_ret_addr);
 
   prot_domains_set_wp(false);
 
-  prot_domains_kern_data[from_id].flags &= ~PROT_DOMAINS_FLAG_BUSY;
+  KERN_READL(flags, prot_domains_kern_data[from_id].flags);
+  flags &= ~PROT_DOMAINS_FLAG_BUSY;
+  KERN_WRITEL(prot_domains_kern_data[from_id].flags, flags);
 
-  inter_dom_call_stk_ptr--;
+  KERN_WRITEL(inter_dom_call_stk_ptr, loc_call_stk_ptr - 1);
 
   dispatcher_tail(from_id, to_id, intr_stk);
 }
@@ -204,11 +220,13 @@ prot_domains_sysret_dispatcher_impl(interrupt_stack_t *intr_stk)
  * \brief  Lookup the current protection domain.
  * \return Kernel data structure for the current protection domain.
  */
-static volatile dom_kern_data_t *
+static volatile dom_kern_data_t ATTR_KERN_ADDR_SPACE *
 get_current_domain(void)
 {
+  uint32_t loc_call_stk_ptr;
   dom_id_t id;
-  id = inter_dom_call_stk[inter_dom_call_stk_ptr - 1];
+  KERN_READL(loc_call_stk_ptr, inter_dom_call_stk_ptr);
+  KERN_READL(id, inter_dom_call_stk[loc_call_stk_ptr - 1]);
   return prot_domains_kern_data + id;
 }
 /*---------------------------------------------------------------------------*/
@@ -219,9 +237,11 @@ get_current_domain(void)
  * \return    Result of the check as a Boolean value
  */
 static bool
-needs_port_io(volatile dom_kern_data_t *dkd)
+needs_port_io(volatile dom_kern_data_t ATTR_KERN_ADDR_SPACE *dkd)
 {
-  return (dkd->flags & PROT_DOMAINS_FLAG_PIO) == PROT_DOMAINS_FLAG_PIO;
+  uint32_t dkd_flags;
+  KERN_READL(dkd_flags, dkd->flags);
+  return (dkd_flags & PROT_DOMAINS_FLAG_PIO) == PROT_DOMAINS_FLAG_PIO;
 }
 /*---------------------------------------------------------------------------*/
 /* Mark the context parameter as volatile so that writes to it will not get
@@ -236,7 +256,7 @@ gp_fault_handler(volatile struct interrupt_context context)
   uint32_t cs_lim;
   uint8_t opcode;
 
-  volatile dom_kern_data_t *dkd = get_current_domain();
+  volatile dom_kern_data_t ATTR_KERN_ADDR_SPACE *dkd = get_current_domain();
   if (needs_port_io(dkd)) {
     __asm__ __volatile__ (
       "mov %%cs, %0\n\t"

cpu/x86/mm/syscalls-int.h

@@ -74,7 +74,7 @@ extern dom_id_t cur_dom;
     #nm ":\n\t"                                                               \
     /* First, load server protection domain ID into EDX, as required by */    \
     /* prot_domains_syscall_dispatcher: */                                    \
-    "  mov " #dcd ", %edx\n\t"                                                \
+    "  mov %" SEG_KERN "s:" #dcd ", %edx\n\t"                                 \
     SYSCALLS_STUB_EPILOGUE(nm))
 
 void syscalls_int_init(void);

cpu/x86/mm/tss.c

@@ -47,15 +47,20 @@ static segment_desc_t ATTR_BSS_GDT sys_tss_desc;
 void
 tss_init(void)
 {
-  sys_tss.iomap_base = sizeof(sys_tss);
+  segment_desc_t seg_desc;
-  sys_tss.esp2 = ((uint32_t)stacks_int) + STACKS_SIZE_INT;
-  sys_tss.ss2 = GDT_SEL_STK_INT;
-  sys_tss.esp0 = ((uint32_t)stacks_exc) + STACKS_SIZE_EXC;
-  sys_tss.ss0 = GDT_SEL_STK_EXC;
 
-  segment_desc_init(&sys_tss_desc, (uint32_t)&sys_tss, sizeof(sys_tss),
+  /* Initialize TSS */
+  KERN_WRITEW(sys_tss.iomap_base, sizeof(sys_tss));
+  KERN_WRITEL(sys_tss.esp2, ((uint32_t)stacks_int) + STACKS_SIZE_INT);
+  KERN_WRITEL(sys_tss.ss2, GDT_SEL_STK_INT);
+  KERN_WRITEL(sys_tss.esp0, ((uint32_t)stacks_exc) + STACKS_SIZE_EXC);
+  KERN_WRITEL(sys_tss.ss0, GDT_SEL_STK_EXC);
+
+  segment_desc_init(&seg_desc,
+                    KERN_DATA_OFF_TO_PHYS_ADDR(&sys_tss), sizeof(sys_tss),
                     SEG_FLAG(DPL, PRIV_LVL_EXC) |
                     SEG_DESCTYPE_SYS | SEG_TYPE_TSS32_AVAIL);
+  gdt_insert(GDT_IDX_OF_DESC(&sys_tss_desc), seg_desc);
 
   __asm__ __volatile__ (
     "ltr %0"