guix/gnu/packages/patches/icecat-CVE-2015-4513-pt06.patch

From 13b2b587c183e85618868752e05ec46bd5a0af86 Mon Sep 17 00:00:00 2001
From: Jon Coppeard <jcoppeard@mozilla.com>
Date: Tue, 13 Oct 2015 11:09:12 +0200
Subject: [PATCH] Bug 1208665 - r=Waldo a=abillings a=sylvestre

---
 js/public/Utility.h         | 49 +++++++++++++++++++++++++++++++++++++--------
 js/src/ds/LifoAlloc.h       | 13 ++++++------
 js/src/jit/FixedList.h      | 10 +++++----
 js/src/jit/JitAllocPolicy.h | 19 ++++++++++--------
 js/src/jit/LIR.cpp          |  3 +--
 js/src/jit/MIRGenerator.h   |  7 ++++---
 js/src/jit/MIRGraph.cpp     |  2 +-
 js/src/jsalloc.h            | 14 ++++++++++---
 js/src/vm/MallocProvider.h  | 39 ++++++++++++++++--------------------
 js/src/vm/Runtime.h         | 10 +++++----
 10 files changed, 105 insertions(+), 61 deletions(-)

diff --git a/js/public/Utility.h b/js/public/Utility.h
index 40b5d90..6b750c3 100644
--- a/js/public/Utility.h
+++ b/js/public/Utility.h
@@ -217,6 +217,36 @@ static inline char* js_strdup(const char* s)
 
 JS_DECLARE_NEW_METHODS(js_new, js_malloc, static MOZ_ALWAYS_INLINE)
 
+namespace js {
+
+/*
+ * Calculate the number of bytes needed to allocate |numElems| contiguous
+ * instances of type |T|.  Return false if the calculation overflowed.
+ */
+template <typename T>
+MOZ_WARN_UNUSED_RESULT inline bool
+CalculateAllocSize(size_t numElems, size_t* bytesOut)
+{
+    *bytesOut = numElems * sizeof(T);
+    return (numElems & mozilla::tl::MulOverflowMask<sizeof(T)>::value) == 0;
+}
+
+/*
+ * Calculate the number of bytes needed to allocate a single instance of type
+ * |T| followed by |numExtra| contiguous instances of type |Extra|.  Return
+ * false if the calculation overflowed.
+ */
+template <typename T, typename Extra>
+MOZ_WARN_UNUSED_RESULT inline bool
+CalculateAllocSizeWithExtra(size_t numExtra, size_t* bytesOut)
+{
+    *bytesOut = sizeof(T) + numExtra * sizeof(Extra);
+    return (numExtra & mozilla::tl::MulOverflowMask<sizeof(Extra)>::value) == 0 &&
+           *bytesOut >= sizeof(T);
+}
+
+} /* namespace js */
+
 template <class T>
 static MOZ_ALWAYS_INLINE void
 js_delete(T* p)
@@ -242,32 +272,34 @@ template <class T>
 static MOZ_ALWAYS_INLINE T*
 js_pod_malloc()
 {
-    return (T*)js_malloc(sizeof(T));
+    return static_cast<T*>(js_malloc(sizeof(T)));
 }
 
 template <class T>
 static MOZ_ALWAYS_INLINE T*
 js_pod_calloc()
 {
-    return (T*)js_calloc(sizeof(T));
+    return static_cast<T*>(js_calloc(sizeof(T)));
 }
 
 template <class T>
 static MOZ_ALWAYS_INLINE T*
 js_pod_malloc(size_t numElems)
 {
-    if (MOZ_UNLIKELY(numElems & mozilla::tl::MulOverflowMask<sizeof(T)>::value))
+    size_t bytes;
+    if (MOZ_UNLIKELY(!js::CalculateAllocSize<T>(numElems, &bytes)))
         return nullptr;
-    return (T*)js_malloc(numElems * sizeof(T));
+    return static_cast<T*>(js_malloc(bytes));
 }
 
 template <class T>
 static MOZ_ALWAYS_INLINE T*
 js_pod_calloc(size_t numElems)
 {
-    if (MOZ_UNLIKELY(numElems & mozilla::tl::MulOverflowMask<sizeof(T)>::value))
+    size_t bytes;
+    if (MOZ_UNLIKELY(!js::CalculateAllocSize<T>(numElems, &bytes)))
         return nullptr;
-    return (T*)js_calloc(numElems * sizeof(T));
+    return static_cast<T*>(js_calloc(bytes));
 }
 
 template <class T>
@@ -275,9 +307,10 @@ static MOZ_ALWAYS_INLINE T*
 js_pod_realloc(T* prior, size_t oldSize, size_t newSize)
 {
     MOZ_ASSERT(!(oldSize & mozilla::tl::MulOverflowMask<sizeof(T)>::value));
-    if (MOZ_UNLIKELY(newSize & mozilla::tl::MulOverflowMask<sizeof(T)>::value))
+    size_t bytes;
+    if (MOZ_UNLIKELY(!js::CalculateAllocSize<T>(newSize, &bytes)))
         return nullptr;
-    return (T*)js_realloc(prior, newSize * sizeof(T));
+    return static_cast<T*>(js_realloc(prior, bytes));
 }
 
 namespace js {
diff --git a/js/src/ds/LifoAlloc.h b/js/src/ds/LifoAlloc.h
index 9dc68c1..35cdc72 100644
--- a/js/src/ds/LifoAlloc.h
+++ b/js/src/ds/LifoAlloc.h
@@ -310,9 +310,10 @@ class LifoAlloc
     // The caller is responsible for initialization.
     template <typename T>
     T* newArrayUninitialized(size_t count) {
-        if (MOZ_UNLIKELY(count & mozilla::tl::MulOverflowMask<sizeof(T)>::value))
+        size_t bytes;
+        if (MOZ_UNLIKELY(!CalculateAllocSize<T>(count, &bytes)))
             return nullptr;
-        return static_cast<T*>(alloc(sizeof(T) * count));
+        return static_cast<T*>(alloc(bytes));
     }
 
     class Mark {
@@ -527,16 +528,16 @@ class LifoAllocPolicy
     {}
     template <typename T>
     T* pod_malloc(size_t numElems) {
-        if (MOZ_UNLIKELY(numElems & mozilla::tl::MulOverflowMask<sizeof(T)>::value))
+        size_t bytes;
+        if (MOZ_UNLIKELY(!CalculateAllocSize<T>(numElems, &bytes)))
             return nullptr;
-        size_t bytes = numElems * sizeof(T);
         void* p = fb == Fallible ? alloc_.alloc(bytes) : alloc_.allocInfallible(bytes);
         return static_cast<T*>(p);
     }
     template <typename T>
     T* pod_calloc(size_t numElems) {
         T* p = pod_malloc<T>(numElems);
-        if (fb == Fallible && !p)
+        if (MOZ_UNLIKELY(!p))
             return nullptr;
         memset(p, 0, numElems * sizeof(T));
         return p;
@@ -544,7 +545,7 @@ class LifoAllocPolicy
     template <typename T>
     T* pod_realloc(T* p, size_t oldSize, size_t newSize) {
         T* n = pod_malloc<T>(newSize);
-        if (fb == Fallible && !n)
+        if (MOZ_UNLIKELY(!n))
             return nullptr;
         MOZ_ASSERT(!(oldSize & mozilla::tl::MulOverflowMask<sizeof(T)>::value));
         memcpy(n, p, Min(oldSize * sizeof(T), newSize * sizeof(T)));
diff --git a/js/src/jit/FixedList.h b/js/src/jit/FixedList.h
index 9cea3a8..b6b37bb 100644
--- a/js/src/jit/FixedList.h
+++ b/js/src/jit/FixedList.h
@@ -37,9 +37,10 @@ class FixedList
         if (length == 0)
             return true;
 
-        if (MOZ_UNLIKELY(length & mozilla::tl::MulOverflowMask<sizeof(T)>::value))
+        size_t bytes;
+        if (MOZ_UNLIKELY(!CalculateAllocSize<T>(length, &bytes)))
             return false;
-        list_ = (T*)alloc.allocate(length * sizeof(T));
+        list_ = (T*)alloc.allocate(bytes);
         return list_ != nullptr;
     }
 
@@ -60,9 +61,10 @@ class FixedList
         size_t newlength = length_ + num;
         if (newlength < length_)
             return false;
-        if (MOZ_UNLIKELY(newlength & mozilla::tl::MulOverflowMask<sizeof(T)>::value))
+        size_t bytes;
+        if (MOZ_UNLIKELY(!CalculateAllocSize<T>(newlength, &bytes)))
             return false;
-        T* list = (T*)alloc.allocate((length_ + num) * sizeof(T));
+        T* list = (T*)alloc.allocate(bytes);
         if (MOZ_UNLIKELY(!list))
             return false;
 
diff --git a/js/src/jit/JitAllocPolicy.h b/js/src/jit/JitAllocPolicy.h
index 4bbd1a3..fca4b3f 100644
--- a/js/src/jit/JitAllocPolicy.h
+++ b/js/src/jit/JitAllocPolicy.h
@@ -48,12 +48,13 @@ class TempAllocator
         return p;
     }
 
-    template <size_t ElemSize>
-    void* allocateArray(size_t n)
+    template <typename T>
+    T* allocateArray(size_t n)
     {
-        if (MOZ_UNLIKELY(n & mozilla::tl::MulOverflowMask<ElemSize>::value))
+        size_t bytes;
+        if (MOZ_UNLIKELY(!CalculateAllocSize<T>(n, &bytes)))
             return nullptr;
-        void* p = lifoScope_.alloc().alloc(n * ElemSize);
+        T* p = static_cast<T*>(lifoScope_.alloc().alloc(bytes));
         if (MOZ_UNLIKELY(!ensureBallast()))
             return nullptr;
         return p;
@@ -79,9 +80,10 @@ class JitAllocPolicy
     {}
     template <typename T>
     T* pod_malloc(size_t numElems) {
-        if (MOZ_UNLIKELY(numElems & mozilla::tl::MulOverflowMask<sizeof(T)>::value))
+        size_t bytes;
+        if (MOZ_UNLIKELY(!CalculateAllocSize<T>(numElems, &bytes)))
             return nullptr;
-        return static_cast<T*>(alloc_.allocate(numElems * sizeof(T)));
+        return static_cast<T*>(alloc_.allocate(bytes));
     }
     template <typename T>
     T* pod_calloc(size_t numElems) {
@@ -112,9 +114,10 @@ class OldJitAllocPolicy
     {}
     template <typename T>
     T* pod_malloc(size_t numElems) {
-        if (MOZ_UNLIKELY(numElems & mozilla::tl::MulOverflowMask<sizeof(T)>::value))
+        size_t bytes;
+        if (MOZ_UNLIKELY(!CalculateAllocSize<T>(numElems, &bytes)))
             return nullptr;
-        return static_cast<T*>(GetJitContext()->temp->allocate(numElems * sizeof(T)));
+        return static_cast<T*>(GetJitContext()->temp->allocate(bytes));
     }
     void free_(void* p) {
     }
diff --git a/js/src/jit/LIR.cpp b/js/src/jit/LIR.cpp
index 70a3fc0..a76e742 100644
--- a/js/src/jit/LIR.cpp
+++ b/js/src/jit/LIR.cpp
@@ -105,8 +105,7 @@ LBlock::init(TempAllocator& alloc)
 
         int numPhis = (phi->type() == MIRType_Value) ? BOX_PIECES : 1;
         for (int i = 0; i < numPhis; i++) {
-            void* array = alloc.allocateArray<sizeof(LAllocation)>(numPreds);
-            LAllocation* inputs = static_cast<LAllocation*>(array);
+            LAllocation* inputs = alloc.allocateArray<LAllocation>(numPreds);
             if (!inputs)
                 return false;
 
diff --git a/js/src/jit/MIRGenerator.h b/js/src/jit/MIRGenerator.h
index 01de27d..5e6b9ef 100644
--- a/js/src/jit/MIRGenerator.h
+++ b/js/src/jit/MIRGenerator.h
@@ -60,10 +60,11 @@ class MIRGenerator
     }
 
     template <typename T>
-    T * allocate(size_t count = 1) {
-        if (count & mozilla::tl::MulOverflowMask<sizeof(T)>::value)
+    T* allocate(size_t count = 1) {
+        size_t bytes;
+        if (MOZ_UNLIKELY(!CalculateAllocSize<T>(count, &bytes)))
             return nullptr;
-        return reinterpret_cast<T*>(alloc().allocate(sizeof(T) * count));
+        return static_cast<T*>(alloc().allocate(bytes));
     }
 
     // Set an error state and prints a message. Returns false so errors can be
diff --git a/js/src/jit/MIRGraph.cpp b/js/src/jit/MIRGraph.cpp
index 5d000dca..4c5cf8e 100644
--- a/js/src/jit/MIRGraph.cpp
+++ b/js/src/jit/MIRGraph.cpp
@@ -297,7 +297,7 @@ MBasicBlock::NewAsmJS(MIRGraph& graph, CompileInfo& info, MBasicBlock* pred, Kin
             size_t nphis = block->stackPosition_;
 
             TempAllocator& alloc = graph.alloc();
-            MPhi* phis = (MPhi*)alloc.allocateArray<sizeof(MPhi)>(nphis);
+            MPhi* phis = alloc.allocateArray<MPhi>(nphis);
             if (!phis)
                 return nullptr;
 
diff --git a/js/src/jsalloc.h b/js/src/jsalloc.h
index ce11ade..e20fa5f2 100644
--- a/js/src/jsalloc.h
+++ b/js/src/jsalloc.h
@@ -53,6 +53,14 @@ class TempAllocPolicy
      */
     JS_FRIEND_API(void*) onOutOfMemory(void* p, size_t nbytes);
 
+    template <typename T>
+    T* onOutOfMemoryTyped(void* p, size_t numElems) {
+        size_t bytes;
+        if (MOZ_UNLIKELY(!CalculateAllocSize<T>(numElems, &bytes)))
+            return nullptr;
+        return static_cast<T*>(onOutOfMemory(p, bytes));
+    }
+
   public:
     MOZ_IMPLICIT TempAllocPolicy(JSContext* cx) : cx_((ContextFriendFields*) cx) {} // :(
     MOZ_IMPLICIT TempAllocPolicy(ContextFriendFields* cx) : cx_(cx) {}
@@ -61,7 +69,7 @@ class TempAllocPolicy
     T* pod_malloc(size_t numElems) {
         T* p = js_pod_malloc<T>(numElems);
         if (MOZ_UNLIKELY(!p))
-            p = static_cast<T*>(onOutOfMemory(nullptr, numElems * sizeof(T)));
+            p = onOutOfMemoryTyped<T>(nullptr, numElems);
         return p;
     }
 
@@ -69,7 +77,7 @@ class TempAllocPolicy
     T* pod_calloc(size_t numElems) {
         T* p = js_pod_calloc<T>(numElems);
         if (MOZ_UNLIKELY(!p))
-            p = static_cast<T*>(onOutOfMemory(reinterpret_cast<void*>(1), numElems * sizeof(T)));
+            p = onOutOfMemoryTyped<T>(reinterpret_cast<void*>(1), numElems);
         return p;
     }
 
@@ -77,7 +85,7 @@ class TempAllocPolicy
     T* pod_realloc(T* prior, size_t oldSize, size_t newSize) {
         T* p2 = js_pod_realloc<T>(prior, oldSize, newSize);
         if (MOZ_UNLIKELY(!p2))
-            p2 = static_cast<T*>(onOutOfMemory(p2, newSize * sizeof(T)));
+            p2 = onOutOfMemoryTyped<T>(p2, newSize);
         return p2;
     }
 
diff --git a/js/src/vm/MallocProvider.h b/js/src/vm/MallocProvider.h
index 1ea4ce2..f334eb1 100644
--- a/js/src/vm/MallocProvider.h
+++ b/js/src/vm/MallocProvider.h
@@ -64,30 +64,27 @@ struct MallocProvider
             client()->updateMallocCounter(numElems * sizeof(T));
             return p;
         }
-        if (numElems & mozilla::tl::MulOverflowMask<sizeof(T)>::value) {
+        size_t bytes;
+        if (MOZ_UNLIKELY(!CalculateAllocSize<T>(numElems, &bytes))) {
             client()->reportAllocationOverflow();
             return nullptr;
         }
-        return (T*)client()->onOutOfMemory(nullptr, numElems * sizeof(T));
+        return static_cast<T*>(client()->onOutOfMemory(nullptr, bytes));
     }
 
     template <class T, class U>
     T* pod_malloc_with_extra(size_t numExtra) {
-        if (MOZ_UNLIKELY(numExtra & mozilla::tl::MulOverflowMask<sizeof(U)>::value)) {
+        size_t bytes;
+        if (MOZ_UNLIKELY((!CalculateAllocSizeWithExtra<T, U>(numExtra, &bytes)))) {
             client()->reportAllocationOverflow();
             return nullptr;
         }
-        size_t bytes = sizeof(T) + numExtra * sizeof(U);
-        if (MOZ_UNLIKELY(bytes < sizeof(T))) {
-            client()->reportAllocationOverflow();
-            return nullptr;
-        }
-        T* p = reinterpret_cast<T*>(js_pod_malloc<uint8_t>(bytes));
+        T* p = static_cast<T*>(js_malloc(bytes));
         if (MOZ_LIKELY(p)) {
             client()->updateMallocCounter(bytes);
             return p;
         }
-        return (T*)client()->onOutOfMemory(nullptr, bytes);
+        return static_cast<T*>(client()->onOutOfMemory(nullptr, bytes));
     }
 
     template <class T>
@@ -108,30 +105,27 @@ struct MallocProvider
             client()->updateMallocCounter(numElems * sizeof(T));
             return p;
         }
-        if (numElems & mozilla::tl::MulOverflowMask<sizeof(T)>::value) {
+        size_t bytes;
+        if (MOZ_UNLIKELY(!CalculateAllocSize<T>(numElems, &bytes))) {
             client()->reportAllocationOverflow();
             return nullptr;
         }
-        return (T*)client()->onOutOfMemory(nullptr, numElems * sizeof(T));
+        return static_cast<T*>(client()->onOutOfMemory(nullptr, bytes));
     }
 
     template <class T, class U>
     T* pod_calloc_with_extra(size_t numExtra) {
-        if (MOZ_UNLIKELY(numExtra & mozilla::tl::MulOverflowMask<sizeof(U)>::value)) {
-            client()->reportAllocationOverflow();
-            return nullptr;
-        }
-        size_t bytes = sizeof(T) + numExtra * sizeof(U);
-        if (MOZ_UNLIKELY(bytes < sizeof(T))) {
+        size_t bytes;
+        if (MOZ_UNLIKELY((!CalculateAllocSizeWithExtra<T, U>(numExtra, &bytes)))) {
             client()->reportAllocationOverflow();
             return nullptr;
         }
-        T* p = reinterpret_cast<T*>(js_pod_calloc<uint8_t>(bytes));
+        T* p = static_cast<T*>(js_calloc(bytes));
         if (MOZ_LIKELY(p)) {
             client()->updateMallocCounter(bytes);
             return p;
         }
-        return (T*)client()->onOutOfMemory(nullptr, bytes);
+        return static_cast<T*>(client()->onOutOfMemory(nullptr, bytes));
     }
 
     template <class T>
@@ -151,11 +145,12 @@ struct MallocProvider
                 client()->updateMallocCounter((newSize - oldSize) * sizeof(T));
             return p;
         }
-        if (newSize & mozilla::tl::MulOverflowMask<sizeof(T)>::value) {
+        size_t bytes;
+        if (MOZ_UNLIKELY(!CalculateAllocSize<T>(newSize, &bytes))) {
             client()->reportAllocationOverflow();
             return nullptr;
         }
-        return (T*)client()->onOutOfMemory(prior, newSize * sizeof(T));
+        return static_cast<T*>(client()->onOutOfMemory(prior, bytes));
     }
 
     JS_DECLARE_NEW_METHODS(new_, pod_malloc<uint8_t>, MOZ_ALWAYS_INLINE)
diff --git a/js/src/vm/Runtime.h b/js/src/vm/Runtime.h
index 90771d6..24c34d3 100644
--- a/js/src/vm/Runtime.h
+++ b/js/src/vm/Runtime.h
@@ -1354,11 +1354,12 @@ struct JSRuntime : public JS::shadow::Runtime,
         T* p = pod_calloc<T>(numElems);
         if (MOZ_LIKELY(!!p))
             return p;
-        if (numElems & mozilla::tl::MulOverflowMask<sizeof(T)>::value) {
+        size_t bytes;
+        if (MOZ_UNLIKELY(!js::CalculateAllocSize<T>(numElems, &bytes))) {
             reportAllocationOverflow();
             return nullptr;
         }
-        return (T*)onOutOfMemoryCanGC(reinterpret_cast<void*>(1), numElems * sizeof(T));
+        return static_cast<T*>(onOutOfMemoryCanGC(reinterpret_cast<void*>(1), bytes));
     }
 
     template <typename T>
@@ -1366,11 +1367,12 @@ struct JSRuntime : public JS::shadow::Runtime,
         T* p2 = pod_realloc<T>(p, oldSize, newSize);
         if (MOZ_LIKELY(!!p2))
             return p2;
-        if (newSize & mozilla::tl::MulOverflowMask<sizeof(T)>::value) {
+        size_t bytes;
+        if (MOZ_UNLIKELY(!js::CalculateAllocSize<T>(newSize, &bytes))) {
             reportAllocationOverflow();
             return nullptr;
         }
-        return (T*)onOutOfMemoryCanGC(p, newSize * sizeof(T));
+        return static_cast<T*>(onOutOfMemoryCanGC(p, bytes));
     }
 
     /*
-- 
2.5.0