use overflow checking macro in a few more places

lib/exec_insn_subr.c

@@ -49,14 +49,14 @@ memory_getptr2(struct exec_context *ctx, uint32_t memidx, uint32_t ptr,
         assert(meminst->allocated <=
                (uint64_t)meminst->size_in_pages
                        << memtype_page_shift(meminst->type));
-        if (__predict_false(offset > UINT32_MAX - ptr)) {
+        uint32_t ea;
+        if (ADD_U32_OVERFLOW(ptr, offset, &ea)) {
                 /*
                  * i failed to find this in the spec.
                  * but some of spec tests seem to test this.
                  */
                 goto do_trap;
         }
-        uint32_t ea = ptr + offset;
         if (__predict_false(size == 0)) {
                 /*
                  * a zero-length access still needs address check.
@@ -70,10 +70,10 @@ memory_getptr2(struct exec_context *ctx, uint32_t memidx, uint32_t ptr,
                 }
                 goto success;
         }
-        if (size - 1 > UINT32_MAX - ea) {
+        uint32_t last_byte = ea + (size - 1);
+        if (ADD_U32_OVERFLOW(ea, size - 1, &last_byte)) {
                 goto do_trap;
         }
-        uint32_t last_byte = ea + (size - 1);
         if (__predict_false(last_byte >= meminst->allocated)) {
                 const uint32_t page_shift = memtype_page_shift(meminst->type);
                 uint32_t need_in_pages = (last_byte >> page_shift) + 1;
@@ -338,16 +338,17 @@ table_grow(struct tableinst *t, const struct val *val, uint32_t n)
         if (n == 0) {
                 return t->size;
         }
-        if (UINT32_MAX - t->size < n || t->size + n > t->type->lim.max) {
+        uint32_t newsize;
+        if (ADD_U32_OVERFLOW(t->size, n, &newsize) ||
+            newsize > t->type->lim.max) {
                 return (uint32_t)-1;
         }
-
-        uint32_t newsize = t->size + n;
         uint32_t csz = valtype_cellsize(t->type->et);
-        size_t newncells = (size_t)newsize * csz;
-        size_t newbytes = newncells * sizeof(*t->cells);
+        size_t newncells;
+        size_t newbytes;
         int ret;
-        if (newbytes / sizeof(*t->cells) / csz != newsize) {
+        if (MUL_SIZE_OVERFLOW((size_t)newsize, (size_t)csz, &newncells) ||
+            MUL_SIZE_OVERFLOW(newncells, sizeof(*t->cells), &newbytes)) {
                 ret = EOVERFLOW;
         } else {
                 size_t oldncells = (size_t)t->size * csz;
@@ -432,11 +433,11 @@ memory_grow_impl(struct exec_context *ctx, struct meminst *mi, uint32_t sz)
 retry:
 #endif
         orig_size = mi->size_in_pages;
-        if (sz > UINT32_MAX - orig_size) {
+        uint32_t new_size;
+        if (ADD_U32_OVERFLOW(orig_size, sz, &new_size)) {
                 memory_unlock(mi);
                 return (uint32_t)-1; /* fail */
         }
-        uint32_t new_size = orig_size + sz;
         assert(lim->max <= WASM_MAX_MEMORY_SIZE >> page_shift);
         if (new_size > lim->max) {
                 memory_unlock(mi);

lib/instance.c

@@ -305,8 +305,8 @@ table_instance_create(struct mem_context *mctx, struct tableinst **tip,
         tinst->mctx = mctx;
         tinst->size = tinst->type->lim.min;
         uint32_t csz = valtype_cellsize(tt->et);
-        size_t ncells = (size_t)tinst->size * csz;
-        if (ncells / csz != tinst->size) {
+        size_t ncells;
+        if (MUL_SIZE_OVERFLOW((size_t)tinst->size, (size_t)csz, &ncells)) {
                 ret = EOVERFLOW;
                 goto fail;
         }

lib/platform.h

@@ -153,7 +153,10 @@
 #if __has_builtin(__builtin_mul_overflow)
 #define MUL_SIZE_OVERFLOW(a, b, c) __builtin_mul_overflow(a, b, c)
 #else
-/* Note: (floor(x) < b) == (x < b) */
+/*
+ * Note: (floor(x) < b) == (x < b) where
+ * x is a real number and b is an integer.
+ */
 #define MUL_SIZE_OVERFLOW(a, b, c)                                            \
         (a != 0 && (SIZE_MAX / a < b) ? 1 : (*c = a * b, 0))
 #endif