Merge pull request monero-project#6 from ndorf/master

Update to Monero v0.17.0.0

common/aligned.c

@@ -1,4 +1,4 @@
-// Copyright (c) 2017-2019, The Monero Project
+// Copyright (c) 2017-2020, The Monero Project
 // 
 // All rights reserved.
 // 

common/apply_permutation.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2017-2019, The Monero Project
+// Copyright (c) 2017-2020, The Monero Project
 // 
 // All rights reserved.
 // 

common/base58.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2019, The Monero Project
+// Copyright (c) 2014-2020, The Monero Project
 // 
 // All rights reserved.
 // 

common/base58.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2019, The Monero Project
+// Copyright (c) 2014-2020, The Monero Project
 // 
 // All rights reserved.
 // 

common/pod-class.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2019, The Monero Project
+// Copyright (c) 2014-2020, The Monero Project
 // 
 // All rights reserved.
 // 

common/threadpool.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2017-2019, The Monero Project
+// Copyright (c) 2017-2020, The Monero Project
 //
 // All rights reserved.
 //
@@ -37,16 +37,14 @@ static __thread bool is_leaf = false;
 namespace tools
 {
 threadpool::threadpool(unsigned int max_threads) : running(true), active(0) {
-  boost::thread::attributes attrs;
-  attrs.set_stack_size(THREAD_STACK_SIZE);
-  max = max_threads ? max_threads : tools::get_max_concurrency();
-  size_t i = max ? max - 1 : 0;
-  while(i--) {
-    threads.push_back(boost::thread(attrs, boost::bind(&threadpool::run, this, false)));
-  }
+  create(max_threads);
 }
 
 threadpool::~threadpool() {
+  destroy();
+}
+
+void threadpool::destroy() {
   try
   {
     const boost::unique_lock<boost::mutex> lock(mutex);
@@ -64,6 +62,24 @@ threadpool::~threadpool() {
     try { threads[i].join(); }
     catch (...) { /* ignore */ }
   }
+  threads.clear();
+}
+
+void threadpool::recycle() {
+  destroy();
+  create(max);
+}
+
+void threadpool::create(unsigned int max_threads) {
+  const boost::unique_lock<boost::mutex> lock(mutex);
+  boost::thread::attributes attrs;
+  attrs.set_stack_size(THREAD_STACK_SIZE);
+  max = max_threads ? max_threads : tools::get_max_concurrency();
+  size_t i = max ? max - 1 : 0;
+  running = true;
+  while(i--) {
+    threads.push_back(boost::thread(attrs, boost::bind(&threadpool::run, this, false)));
+  }
 }
 
 void threadpool::submit(waiter *obj, std::function<void()> f, bool leaf) {
@@ -104,20 +120,20 @@ threadpool::waiter::~waiter()
   catch (...) { /* ignore */ }
   try
   {
-    wait(NULL);
+    wait();
   }
   catch (const std::exception &e)
   {
     /* ignored */
   }
 }
 
-void threadpool::waiter::wait(threadpool *tpool) {
-  if (tpool)
-    tpool->run(true);
+bool threadpool::waiter::wait() {
+  pool.run(true);
   boost::unique_lock<boost::mutex> lock(mt);
   while(num)
     cv.wait(lock);
+  return !error();
 }
 
 void threadpool::waiter::inc() {
@@ -145,12 +161,13 @@ void threadpool::run(bool flush) {
     if (!running) break;
 
     active++;
-    e = queue.front();
+    e = std::move(queue.front());
     queue.pop_front();
     lock.unlock();
     ++depth;
     is_leaf = e.leaf;
-    e.f();
+    try { e.f(); }
+    catch (const std::exception &ex) { e.wo->set_error(); try { MERROR("Exception in threadpool job: " << ex.what()); } catch (...) {} }
     --depth;
     is_leaf = false;
 

common/threadpool.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2017-2019, The Monero Project
+// Copyright (c) 2017-2020, The Monero Project
 //
 // All rights reserved.
 //
@@ -55,12 +55,16 @@ class threadpool
   class waiter {
     boost::mutex mt;
     boost::condition_variable cv;
+    threadpool &pool;
     int num;
+    bool error_flag;
     public:
     void inc();
     void dec();
-    void wait(threadpool *tpool);  //! Wait for a set of tasks to finish.
-    waiter() : num(0){}
+    bool wait();  //! Wait for a set of tasks to finish, returns false iff any error
+    void set_error() noexcept { error_flag = true; }
+    bool error() const noexcept { return error_flag; }
+    waiter(threadpool &pool) : pool(pool), num(0), error_flag(false) {}
     ~waiter();
   };
 
@@ -69,12 +73,17 @@ class threadpool
   // task to finish.
   void submit(waiter *waiter, std::function<void()> f, bool leaf = false);
 
+  // destroy and recreate threads
+  void recycle();
+
   unsigned int get_max_concurrency() const;
 
   ~threadpool();
 
   private:
     threadpool(unsigned int max_threads = 0);
+    void destroy();
+    void create(unsigned int max_threads);
     typedef struct entry {
       waiter *wo;
       std::function<void()> f;

common/utf8.h

@@ -0,0 +1,114 @@
+// Copyright (c) 2019, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once 
+
+#include <cctype>
+#include <cwchar>
+#include <stdexcept>
+
+namespace tools
+{
+  template<typename T, typename Transform>
+  inline T utf8canonical(const T &s, Transform t = [](wint_t c)->wint_t { return c; })
+  {
+    T sc = "";
+    size_t avail = s.size();
+    const char *ptr = s.data();
+    wint_t cp = 0;
+    int bytes = 1;
+    char wbuf[8], *wptr;
+    while (avail--)
+    {
+      if ((*ptr & 0x80) == 0)
+      {
+        cp = *ptr++;
+        bytes = 1;
+      }
+      else if ((*ptr & 0xe0) == 0xc0)
+      {
+        if (avail < 1)
+          throw std::runtime_error("Invalid UTF-8");
+        cp = (*ptr++ & 0x1f) << 6;
+        cp |= *ptr++ & 0x3f;
+        --avail;
+        bytes = 2;
+      }
+      else if ((*ptr & 0xf0) == 0xe0)
+      {
+        if (avail < 2)
+          throw std::runtime_error("Invalid UTF-8");
+        cp = (*ptr++ & 0xf) << 12;
+        cp |= (*ptr++ & 0x3f) << 6;
+        cp |= *ptr++ & 0x3f;
+        avail -= 2;
+        bytes = 3;
+      }
+      else if ((*ptr & 0xf8) == 0xf0)
+      {
+        if (avail < 3)
+          throw std::runtime_error("Invalid UTF-8");
+        cp = (*ptr++ & 0x7) << 18;
+        cp |= (*ptr++ & 0x3f) << 12;
+        cp |= (*ptr++ & 0x3f) << 6;
+        cp |= *ptr++ & 0x3f;
+        avail -= 3;
+        bytes = 4;
+      }
+      else
+        throw std::runtime_error("Invalid UTF-8");
+
+      cp = t(cp);
+      if (cp <= 0x7f)
+        bytes = 1;
+      else if (cp <= 0x7ff)
+        bytes = 2;
+      else if (cp <= 0xffff)
+        bytes = 3;
+      else if (cp <= 0x10ffff)
+        bytes = 4;
+      else
+        throw std::runtime_error("Invalid code point UTF-8 transformation");
+
+      wptr = wbuf;
+      switch (bytes)
+      {
+        case 1: *wptr++ = cp; break;
+        case 2: *wptr++ = 0xc0 | (cp >> 6); *wptr++ = 0x80 | (cp & 0x3f); break;
+        case 3: *wptr++ = 0xe0 | (cp >> 12); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
+        case 4: *wptr++ = 0xf0 | (cp >> 18); *wptr++ = 0x80 | ((cp >> 12) & 0x3f); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
+        default: throw std::runtime_error("Invalid UTF-8");
+      }
+      *wptr = 0;
+      sc.append(wbuf, bytes);
+      cp = 0;
+      bytes = 1;
+    }
+    return sc;
+  }
+}

common/util.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2019, The Monero Project
+// Copyright (c) 2014-2020, The Monero Project
 // 
 // All rights reserved.
 // 
@@ -35,32 +35,17 @@
 
 #include "util.h"
 
-#include <boost/thread/thread.hpp>
 
 namespace tools
 {
 
-  namespace
-  {
-    boost::mutex max_concurrency_lock;
-    unsigned max_concurrency = boost::thread::hardware_concurrency();
-  }
-
   void set_max_concurrency(unsigned n)
   {
-    if (n < 1)
-      n = boost::thread::hardware_concurrency();
-    unsigned hwc = boost::thread::hardware_concurrency();
-    if (n > hwc)
-      n = hwc;
-    boost::lock_guard<boost::mutex> lock(max_concurrency_lock);
-    max_concurrency = n;
   }
 
   unsigned get_max_concurrency()
   {
-    boost::lock_guard<boost::mutex> lock(max_concurrency_lock);
-    return max_concurrency;
+    return 1;
   }
 
 

common/util.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2019, The Monero Project
+// Copyright (c) 2014-2020, The Monero Project
 // 
 // All rights reserved.
 // 

common/varint.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2019, The Monero Project
+// Copyright (c) 2014-2020, The Monero Project
 // 
 // All rights reserved.
 // 

crypto/blake256.c

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2019, The Monero Project
+// Copyright (c) 2014-2020, The Monero Project
 // 
 // All rights reserved.
 // 
@@ -40,6 +40,7 @@
 #include <string.h>
 #include <stdio.h>
 #include <stdint.h>
+#include <memwipe.h>
 #include "blake256.h"
 
 #define U8TO32(p) \
@@ -277,7 +278,7 @@ void hmac_blake256_init(hmac_state *S, const uint8_t *_key, uint64_t keylen) {
     }
     blake256_update(&S->outer, pad, 512);
 
-    memset(keyhash, 0, 32);
+    memwipe(keyhash, sizeof(keyhash));
 }
 
 // keylen = number of bytes
@@ -307,7 +308,7 @@ void hmac_blake224_init(hmac_state *S, const uint8_t *_key, uint64_t keylen) {
     }
     blake224_update(&S->outer, pad, 512);
 
-    memset(keyhash, 0, 32);
+    memwipe(keyhash, sizeof(keyhash));
 }
 
 // datalen = number of bits
@@ -327,15 +328,15 @@ void hmac_blake256_final(hmac_state *S, uint8_t *digest) {
     blake256_final(&S->inner, ihash);
     blake256_update(&S->outer, ihash, 256);
     blake256_final(&S->outer, digest);
-    memset(ihash, 0, 32);
+    memwipe(ihash, sizeof(ihash));
 }
 
 void hmac_blake224_final(hmac_state *S, uint8_t *digest) {
     uint8_t ihash[32];
     blake224_final(&S->inner, ihash);
     blake224_update(&S->outer, ihash, 224);
     blake224_final(&S->outer, digest);
-    memset(ihash, 0, 32);
+    memwipe(ihash, sizeof(ihash));
 }
 
 // keylen = number of bytes; inlen = number of bytes

crypto/blake256.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2019, The Monero Project
+// Copyright (c) 2014-2020, The Monero Project
 // 
 // All rights reserved.
 // 

crypto/chacha.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2019, The Monero Project
+// Copyright (c) 2014-2020, The Monero Project
 // 
 // All rights reserved.
 //