🐧🐎add likely/unlikely and fix synchronous invocation bugs & performan…

…ce boost

MEukaron/Include/Kernel/captbl.h

@@ -3,7 +3,11 @@ Filename    : captbl.h
 Author      : pry
 Date        : 01/04/2017
 Licence     : LGPL v3+; see COPYING for details.
-Description : The header of type table.
+Description : The header of type table. All the branches in these macros are
+              pretty much optimized with RME_LIKELY and (mainly )RME_UNLIKELY.
+              Due to the huge penalty of a wrong hint, only error handling in
+              headers, synchronous invocation activation and synchronous invocation
+              returning are heavily optimized.
 ******************************************************************************/
 
 /* Defines *******************************************************************/
@@ -114,10 +118,10 @@ while(0)
 do \
 { \
     /* See if the capability is frozen */ \
-    if(((CAP)->Head.Type_Ref&RME_CAP_FROZEN)!=0) \
+    if(RME_UNLIKELY(((CAP)->Head.Type_Ref&RME_CAP_FROZEN)!=0)) \
         return RME_ERR_CAP_FROZEN; \
     /* See if this capability allows such operations */ \
-    if(((CAP)->Head.Flags&(FLAG))!=(FLAG)) \
+    if(RME_UNLIKELY(((CAP)->Head.Flags&(FLAG))!=(FLAG))) \
         return RME_ERR_CAP_FLAG; \
 } \
 while(0)
@@ -127,41 +131,41 @@ while(0)
 do \
 { \
     /* See if the creation of such capability is allowed */ \
-    if(((CAP)->Head.Flags&(FLAG))!=(FLAG)) \
+    if(RME_UNLIKELY(((CAP)->Head.Flags&(FLAG))!=(FLAG))) \
         return RME_ERR_CAP_FLAG; \
     /* The end is always aligned to 256 bytes in the kernel, and does not include the ending byte */ \
-    if(((CAP)->Start>(START))||((CAP)->End<((START)+(SIZE)))) \
+    if(RME_UNLIKELY(((CAP)->Start>(START))||((CAP)->End<((START)+(SIZE))))) \
         return RME_ERR_CAP_FLAG; \
 } \
 while(0)
 
-/* Defrost a frozen cap */
+/* Defrost a frozen cap - we do not check failure because if we fail, someone have done it for us */
 #define RME_CAP_DEFROST(CAP,TEMP) \
 do \
 { \
     __RME_Comp_Swap(&((CAP)->Head.Type_Ref),&(TEMP),(TEMP)&(~((ptr_t)RME_CAP_FROZEN))); \
 } \
 while(0)
-    
+
 /* Checks to be done before deleting */
 #define RME_CAP_DEL_CHECK(CAP,TEMP,TYPE) \
 do \
 { \
     (TEMP)=(CAP)->Head.Type_Ref; \
-    /* See if the slot is frozen, and its cap must be not zero */ \
-    if(((TEMP)&RME_CAP_FROZEN)!=0) \
+    /* See if the slot is frozen, and its cap must be non-zero */ \
+    if(RME_UNLIKELY(((TEMP)&RME_CAP_FROZEN)!=0)) \
         return RME_ERR_CAP_FROZEN; \
     /* See if we are in the creation/delegation process - This frozen flag is set by the creator */ \
-    if(RME_CAP_TYPE(TEMP)==RME_CAP_NOP) \
+    if(RME_UNLIKELY(RME_CAP_TYPE(TEMP)==RME_CAP_NOP)) \
         return RME_ERR_CAP_NULL; \
     /* See if the cap type is correct. Only deletion checks type, while removing does not */ \
-    if(RME_CAP_TYPE(TEMP)!=(TYPE)) \
+    if(RME_UNLIKELY(RME_CAP_TYPE(TEMP)!=(TYPE))) \
         return RME_ERR_CAP_TYPE; \
     /* See if the slot is quiescent */ \
-    if(RME_CAP_QUIE((CAP)->Head.Timestamp)==0) \
+    if(RME_UNLIKELY(RME_CAP_QUIE((CAP)->Head.Timestamp)==0)) \
         return RME_ERR_CAP_QUIE; \
     /* To use deletion, we must be an unreferenced root */ \
-    if((RME_CAP_REF(TEMP)!=0)||(((CAP)->Head.Parent)!=0)) \
+    if(RME_UNLIKELY((RME_CAP_REF(TEMP)!=0)||(((CAP)->Head.Parent)!=0))) \
     { \
         /* We defrost the cap and return. Need cas, in case two competing deletions happen */ \
         RME_CAP_DEFROST(CAP,TEMP); \
@@ -175,17 +179,17 @@ while(0)
 do \
 { \
     (TEMP)=(CAP)->Head.Type_Ref; \
-    /* See if the slot is frozen, and its cap must be not zero */ \
-    if(((TEMP)&RME_CAP_FROZEN)!=0) \
+    /* See if the slot is frozen, and its cap must be non-zero */ \
+    if(RME_UNLIKELY(((TEMP)&RME_CAP_FROZEN)!=0)) \
         return RME_ERR_CAP_FROZEN; \
     /* See if we are in the creation/delegation process - This frozen flag is set by the creator */ \
-    if(RME_CAP_TYPE(TEMP)==RME_CAP_NOP) \
+    if(RME_UNLIKELY(RME_CAP_TYPE(TEMP)==RME_CAP_NOP)) \
         return RME_ERR_CAP_NULL; \
     /* See if the slot is quiescent */ \
-    if(RME_CAP_QUIE((CAP)->Head.Timestamp)==0) \
+    if(RME_UNLIKELY(RME_CAP_QUIE((CAP)->Head.Timestamp)==0)) \
         return RME_ERR_CAP_QUIE; \
     /* To use removal, we must be an unreferenced child */ \
-    if((RME_CAP_REF(TEMP)!=0)||(((CAP)->Head.Parent)==0)) \
+    if(RME_UNLIKELY((RME_CAP_REF(TEMP)!=0)||(((CAP)->Head.Parent)==0))) \
     { \
         /* We defrost the cap and return. Need cas, in case two competing removals happen */ \
         RME_CAP_DEFROST(CAP,TEMP); \
@@ -199,7 +203,7 @@ while(0)
 do \
 { \
     /* If this fails, then it means that somebody have deleted/removed it first */ \
-    if(__RME_Comp_Swap(&((CAP)->Head.Type_Ref),&(TEMP),0)==0) \
+    if(RME_UNLIKELY(__RME_Comp_Swap(&((CAP)->Head.Type_Ref),&(TEMP),0)==0)) \
         return RME_ERR_CAP_NULL; \
 } \
 while(0)
@@ -210,7 +214,7 @@ do \
 { \
     /* Check if anything is there. If there is nothing there, the Type_Ref must be 0 */ \
     (TEMP)=RME_CAP_TYPEREF(RME_CAP_NOP,0); \
-    if(__RME_Comp_Swap(&((CAP)->Head.Type_Ref),&(TEMP),RME_CAP_FROZEN)==0) \
+    if(RME_UNLIKELY(__RME_Comp_Swap(&((CAP)->Head.Type_Ref),&(TEMP),RME_CAP_FROZEN)==0)) \
         return RME_ERR_CAP_EXIST; \
 } \
 while(0)
@@ -220,7 +224,7 @@ while(0)
 do \
 { \
     /* Check if the captbl is over range */ \
-    if((CAP_NUM)>=((CAPTBL)->Entry_Num)) \
+    if(RME_UNLIKELY((CAP_NUM)>=((CAPTBL)->Entry_Num))) \
         return RME_ERR_CAP_RANGE; \
     /* Get the slot position */ \
     (PARAM)=&(RME_CAP_GETOBJ((CAPTBL),TYPE)[(CAP_NUM)]); \
@@ -235,35 +239,35 @@ do \
     if(((CAP_NUM)&RME_CAPID_2L)==0) \
     { \
         /* Check if the captbl is over range */ \
-        if((CAP_NUM)>=((CAPTBL)->Entry_Num)) \
+        if(RME_UNLIKELY((CAP_NUM)>=((CAPTBL)->Entry_Num))) \
             return RME_ERR_CAP_RANGE; \
         /* Get the cap slot and check the type */ \
         (PARAM)=(TYPE)(&RME_CAP_GETOBJ(CAPTBL,struct RME_Cap_Struct*)[(CAP_NUM)]); \
-        if(RME_CAP_TYPE((PARAM)->Head.Type_Ref)!=(CAP_TYPE)) \
+        if(RME_UNLIKELY(RME_CAP_TYPE((PARAM)->Head.Type_Ref)!=(CAP_TYPE))) \
             return RME_ERR_CAP_TYPE; \
     } \
     /* Yes, this is a 2-level cap */ \
     else \
     { \
         /* Check if the cap to potential captbl is over range */ \
-        if(RME_CAP_H(CAP_NUM)>=((CAPTBL)->Entry_Num)) \
+        if(RME_UNLIKELY(RME_CAP_H(CAP_NUM)>=((CAPTBL)->Entry_Num))) \
             return RME_ERR_CAP_RANGE; \
         /* Get the cap slot */ \
         (PARAM)=(TYPE)(&RME_CAP_GETOBJ(CAPTBL,struct RME_Cap_Captbl*)[RME_CAP_H(CAP_NUM)]); \
         \
         /* See if the captbl is frozen for deletion or removal */ \
-        if(((PARAM)->Head.Type_Ref&RME_CAP_FROZEN)!=0) \
+        if(RME_UNLIKELY(((PARAM)->Head.Type_Ref&RME_CAP_FROZEN)!=0)) \
             return RME_ERR_CAP_FROZEN; \
         /* See if this is a captbl */ \
-        if(RME_CAP_TYPE((PARAM)->Head.Type_Ref)!=RME_CAP_CAPTBL) \
+        if(RME_UNLIKELY(RME_CAP_TYPE((PARAM)->Head.Type_Ref)!=RME_CAP_CAPTBL)) \
             return RME_ERR_CAP_TYPE; \
         \
         /* Check if the 2nd-layer captbl is over range */ \
-        if(RME_CAP_L(CAP_NUM)>=(((struct RME_Cap_Captbl*)(PARAM))->Entry_Num)) \
+        if(RME_UNLIKELY(RME_CAP_L(CAP_NUM)>=(((struct RME_Cap_Captbl*)(PARAM))->Entry_Num))) \
             return RME_ERR_CAP_RANGE; \
         /* Get the cap slot and check the type */ \
         (PARAM)=(TYPE)(&RME_CAP_GETOBJ(PARAM,struct RME_Cap_Struct*)[RME_CAP_L(CAP_NUM)]); \
-        if(RME_CAP_TYPE((PARAM)->Head.Type_Ref)!=(CAP_TYPE)) \
+        if(RME_UNLIKELY(RME_CAP_TYPE((PARAM)->Head.Type_Ref)!=(CAP_TYPE))) \
             return RME_ERR_CAP_TYPE; \
     } \
 } \

MEukaron/Include/Kernel/kernel.h

@@ -95,7 +95,7 @@ Description : The header of kernel system call path.
  * Possible categories of context switch includes synchronous invocation and thread switch. */
 #define RME_SWITCH_RETURN(REG,RETVAL) \
 { \
-    if((RETVAL)<0) \
+    if(RME_UNLIKELY((RETVAL)<0)) \
         __RME_Set_Syscall_Retval((REG),(RETVAL)); \
     \
     return; \
@@ -123,7 +123,7 @@ Description : The header of kernel system call path.
 #define RME_ASSERT(X) \
 do \
 { \
-    if((X)==0) \
+    if(RME_UNLIKELY((X)==0)) \
     { \
         RME_PRINTK_S((s8*)"\r\n***\r\nKernel panic - not syncing:\r\n"); \
         RME_PRINTK_S((s8*)__FILE__); \

MEukaron/Include/Kernel/prcthd.h

@@ -49,7 +49,7 @@ Description : The header of page table.
 do \
 { \
     /* Check if exceeded maximum time or overflowed */ \
-    if((((DST)+(AMOUNT))>=RME_THD_MAX_TIME)||(((DST)+(AMOUNT))<(DST))) \
+    if(RME_UNLIKELY((((DST)+(AMOUNT))>=RME_THD_MAX_TIME)||(((DST)+(AMOUNT))<(DST)))) \
         return RME_ERR_PTH_OVERFLOW; \
 } \
 while(0);
@@ -150,8 +150,8 @@ struct RME_Thd_Regs
 {
     /* The register set - architectural specific */
     struct RME_Reg_Struct Reg;
-    /* The co-processor/peripheral context - architectural specific. This usually
-     * contains the FPU data */
+    /* The co-processor/peripheral context - architecture specific.
+     * This usually contains the FPU data */
     struct RME_Cop_Struct Cop_Reg;
 };
 

MEukaron/Include/Platform/CortexM/platform_cmx.h

@@ -84,6 +84,17 @@ typedef s32 ret_t;
 #define EXTERN                  extern
 /* Compiler "inline" keyword setting */
 #define INLINE                  __forceinline
+/* Compiler likely & unlikely setting */
+#ifdef likely
+#define RME_LIKELY(X)           (likely(X))
+#else
+#define RME_LIKELY(X)           (X)
+#endif
+#ifdef unlikely
+#define RME_UNLIKELY(X)         (unlikely(X))
+#else
+#define RME_UNLIKELY(X)         (X)
+#endif
 /* Number of CPUs in the system */
 #define RME_CPU_NUM             1
 /* The order of bits in one CPU machine word */

MEukaron/Include/Platform/X64/platform_x64.h

@@ -95,6 +95,17 @@ typedef s64 ret_t;
 #define EXTERN                               extern
 /* Compiler "inline" keyword setting */
 #define INLINE                               inline
+/* Compiler likely & unlikely setting */
+#ifdef likely
+#define RME_LIKELY(X)                        (likely(X))
+#else
+#define RME_LIKELY(X)                        (X)
+#endif
+#ifdef unlikely
+#define RME_UNLIKELY(X)                      (unlikely(X))
+#else
+#define RME_UNLIKELY(X)                      (X)
+#endif
 /* Number of CPUs in the system - max. 4096 ones are supported */
 #define RME_CPU_NUM                          256
 /* The order of bits in one CPU machine word */
@@ -223,7 +234,7 @@ typedef s64 ret_t;
 #define RME_X64_MMU_KERN_PDE                 (RME_X64_MMU_P|RME_X64_MMU_PDE_SUP|RME_X64_MMU_RW|RME_X64_MMU_G)
 
 /* MMU definitions */
-/* Write info to MMU */
+/* Write info to MMU - no longer used because we have PCID */
 #define RME_X64_CR3_PCD                      (1<<4)
 #define RME_X64_CR3_PWT                      (1<<3)
 
@@ -812,8 +823,10 @@ struct RME_X64_Features
 /* Page table registration table */
 struct __RME_X64_Pgreg
 {
-    /* How many child page tables does this page table have? */
-    ptr_t Child_Cnt;
+	/* What is the PCID of this page table? - This can be set through kernel function caps */
+	u32 PCID;
+    /* How many child page tables does this page table have? - this can never overflow */
+    u32 Child_Cnt;
     /* How many parent page tables does this page table have? */
     ptr_t Parent_Cnt;
 };
@@ -867,6 +880,8 @@ static volatile struct RME_X64_IOAPIC_Info RME_X64_IOAPIC_Info[8];
 static volatile ptr_t RME_X64_LAPIC_Addr;
 /* The processor features */
 static volatile struct RME_X64_Features RME_X64_Feature;
+/* The PCID counter */
+static volatile ptr_t RME_X64_PCID_Inc;
 
 /* Translate the flags into X64 specific ones - the STATIC bit will never be
  * set thus no need to consider about it here. The flag bits order is shown below:

MEukaron/Kernel/captbl.c

@@ -166,7 +166,6 @@ ret_t _RME_Captbl_Boot_Crt(struct RME_Cap_Captbl* Captbl, cid_t Cap_Captbl_Crt,
     /* Check if the target captbl is not frozen and allows such operations */
     RME_CAP_CHECK(Captbl_Op,RME_CAPTBL_FLAG_CRT);
 
-
     /* Get the cap slot */
     RME_CAPTBL_GETSLOT(Captbl_Op,Cap_Crt,struct RME_Cap_Captbl*,Captbl_Crt);
     /* Take the slot if possible */

MEukaron/Kernel/kernel.c

@@ -292,11 +292,21 @@ void _RME_Svc_Handler(struct RME_Reg_Struct* Reg)
     ptr_t Param[3];
     ret_t Retval;
     ptr_t CPUID;
+    ptr_t Svc_Num;
     struct RME_Inv_Struct* Inv_Top;
     struct RME_Cap_Captbl* Captbl;
-    
+
     /* Get the system call parameters from the system call */
     __RME_Get_Syscall_Param(Reg, &Svc, &Capid, Param);
+    Svc_Num=Svc&0x3F;
+
+    /* Fast path - synchronous invocation returning */
+    if(Svc_Num==RME_SVC_INV_RET)
+    {
+        Retval=_RME_Inv_Ret(Reg /* struct RME_Reg_Struct* Reg */,
+                            0   /* ptr_t Fault_Flag */);
+        RME_SWITCH_RETURN(Reg,Retval);
+    }
 
     /* Get our current capability table. No need to check whether it is frozen
      * because it can't be deleted anyway */
@@ -306,29 +316,24 @@ void _RME_Svc_Handler(struct RME_Reg_Struct* Reg)
         Captbl=RME_Cur_Thd[CPUID]->Sched.Proc->Captbl;
     else
         Captbl=Inv_Top->Proc->Captbl;
+
+    /* Fast path - synchronous invocation activation */
+    if(Svc_Num==RME_SVC_INV_ACT)
+    {
+        Retval=_RME_Inv_Act(Captbl, Reg      /* struct RME_Reg_Struct* Reg */,
+                                    Param[0] /* cid_t Cap_Inv */,
+                                    Param[1] /* ptr_t Param */);
+        RME_SWITCH_RETURN(Reg,Retval);
+    }
 
     /* See if this operation can potentially cause a register set switch. All the 
      * functions that may cause a register set switch is listed here. The behavior
      * of these functions shall be: If the function is successful, they shall
      * perform the return value saving on proper register stacks by themselves;
-     * if the function fails, it should not conduct such return value saving. */
-    switch((Svc&0x3F))
+     * if the function fails, it should not conduct such return value saving. These
+     * paths are less optimized than synchronous invocation, but are still optimized */
+    switch(Svc_Num)
     {
-        /* Return from invocation */
-        case RME_SVC_INV_RET:
-        {
-            Retval=_RME_Inv_Ret(Reg /* struct RME_Reg_Struct* Reg */,
-                                0   /* ptr_t Fault_Flag */);
-            RME_SWITCH_RETURN(Reg,Retval);
-        }
-        /* Activate an invocation */
-        case RME_SVC_INV_ACT:
-        {
-            Retval=_RME_Inv_Act(Captbl, Reg      /* struct RME_Reg_Struct* Reg */,
-                                        Param[0] /* cid_t Cap_Inv */,
-                                        Param[1] /* ptr_t Param */);
-            RME_SWITCH_RETURN(Reg,Retval);
-        }
         /* Send to a signal endpoint */
         case RME_SVC_SIG_SND:
         {
@@ -390,7 +395,7 @@ void _RME_Svc_Handler(struct RME_Reg_Struct* Reg)
     } 
 
     /* It is guaranteed that these functions will never cause a context switch */
-    switch((Svc&0x3F))
+    switch(Svc_Num)
     {
         /* Capability table */
         case RME_SVC_CAPTBL_CRT: