Adds libsecp256k1 installation and addresses reviews

Update no-history-sync code:
This updates the new functionality in jmclient.wallet_utils
in the no-history-sync PR #444 to be compatible
with the python-bitcointx refactoring.

Remove all future/py2 compatibility code remaining:
This is in line with #525 and corrects erroneous
addition of more compatibility code.

Addresses all flake8 complaints (ununsed imports etc)

Addresses review of @dgpv

Addresses review of @kristapsk

install.sh

@@ -234,6 +234,40 @@ libffi_install ()
     popd
 }
 
+libsecp256k1_build()
+{
+    make clean
+    ./autogen.sh
+    ./configure \
+    --enable-module-recovery \
+    --disable-jni \
+    --prefix "${jm_root}" \
+    --enable-experimental \
+    --enable-module-ecdh \
+    --enable-benchmark=no
+    make
+    if ! make check; then
+        return 1
+    fi
+}
+
+libsecp256k1_install()
+{
+    secp256k1_lib_tar='0d9540b13ffcd7cd44cc361b8744b93d88aa76ba'
+    secp256k1_lib_sha="0803d2dddbf6dd702c379118f066f638bcef6b07eea959f12d31ad2f4721fbe1"
+    secp256k1_lib_url='https://github.com/bitcoin-core/secp256k1/archive'
+    if ! dep_get "${secp256k1_lib_tar}.tar.gz" "${secp256k1_lib_sha}" "${secp256k1_lib_url}"; then
+        return 1
+    fi
+    pushd "secp256k1-${secp256k1_lib_tar}"
+    if libsecp256k1_build; then
+        make install
+    else
+        return 1
+    fi
+    popd
+}
+
 libsodium_build ()
 {
     make uninstall
@@ -419,6 +453,10 @@ main ()
 #        echo "Openssl was not built. Exiting."
 #        return 1
 #    fi
+    if ! libsecp256k1_install; then
+        echo "libsecp256k1 was not built. Exiting."
+        return 1
+    fi
     if ! libffi_install; then
         echo "Libffi was not built. Exiting."
         return 1

jmbase/jmbase/support.py

@@ -220,9 +220,9 @@ def print_jm_version(option, opt_str, value, parser):
 # helper functions for conversions of format between over-the-wire JM
 # and internal. See details in hexbin() docstring.
 
-def cv(x):
-    success, utxo = utxostr_to_utxo(x)
-    if success:
+def _convert(x):
+    good, utxo = utxostr_to_utxo(x)
+    if good:
         return utxo
     else:
         try:
@@ -239,18 +239,18 @@ def listchanger(l):
         elif isinstance(x, dict):
             rlist.append(dictchanger(x))
         else:
-            rlist.append(cv(x))
+            rlist.append(_convert(x))
     return rlist
 
 def dictchanger(d):
     rdict = {}
     for k, v in d.items():
         if isinstance(v, dict):
-            rdict[cv(k)] = dictchanger(v)
+            rdict[_convert(k)] = dictchanger(v)
         elif isinstance(v, list):
-            rdict[cv(k)] = listchanger(v)
+            rdict[_convert(k)] = listchanger(v)
         else:
-            rdict[cv(k)] = cv(v)
+            rdict[_convert(k)] = _convert(v)
     return rdict
 
 def hexbin(func):
@@ -276,7 +276,7 @@ def func_wrapper(inst, *args, **kwargs):
             elif isinstance(arg, dict):
                 newargs.append(dictchanger(arg))
             else:
-                newargs.append(cv(arg))
+                newargs.append(_convert(arg))
         return func(inst, *newargs, **kwargs)
 
     return func_wrapper

jmbase/setup.py

@@ -9,7 +9,7 @@
       author_email='',
       license='GPL',
       packages=['jmbase'],
-      install_requires=['future', 'twisted==19.7.0', 'service-identity',
+      install_requires=['twisted==19.7.0', 'service-identity',
                         'chromalog==1.0.5'],
-      python_requires='>=3.3',
+      python_requires='>=3.6',
       zip_safe=False)

jmbitcoin/jmbitcoin/__init__.py

@@ -13,7 +13,5 @@
 from bitcointx.core.key import KeyStore
 from bitcointx.core.script import (CScript, OP_0, SignatureHash, SIGHASH_ALL,
                                    SIGVERSION_WITNESS_V0, CScriptWitness)
-from bitcointx.wallet import (CBitcoinSecret, P2WPKHBitcoinAddress, CCoinAddress,
-                              P2SHCoinAddress)
 from bitcointx.core.psbt import PartiallySignedTransaction
 

jmbitcoin/jmbitcoin/secp256k1_ecies.py

@@ -1,8 +1,5 @@
 #!/usr/bin/python
-from __future__ import (absolute_import, division,
-                        print_function, unicode_literals)
-from builtins import * # noqa: F401
-from future.utils import native
+
 import coincurve as secp256k1
 import base64
 import hmac
@@ -19,15 +16,15 @@ class ECIESDecryptionError(Exception):
 # AES primitives. See BIP-SNICKER for specification.
 def aes_encrypt(key, data, iv):
     encrypter = pyaes.Encrypter(
-        pyaes.AESModeOfOperationCBC(key, iv=native(iv)))
+        pyaes.AESModeOfOperationCBC(key, iv=iv))
     enc_data = encrypter.feed(data)
     enc_data += encrypter.feed()
 
     return enc_data
 
 def aes_decrypt(key, data, iv):
     decrypter = pyaes.Decrypter(
-        pyaes.AESModeOfOperationCBC(key, iv=native(iv)))
+        pyaes.AESModeOfOperationCBC(key, iv=iv))
     try:
         dec_data = decrypter.feed(data)
         dec_data += decrypter.feed()

jmbitcoin/jmbitcoin/secp256k1_main.py

@@ -1,15 +1,11 @@
 #!/usr/bin/python
-from future.utils import native_bytes, bytes_to_native_str
-import binascii
-import hashlib
-import sys
 import base64
 import struct
 import coincurve as secp256k1
 
 from bitcointx import base58
-from bitcointx.core import Hash, CBitcoinTransaction
-from bitcointx.core.key import CKeyBase, CPubKey
+from bitcointx.core import Hash
+from bitcointx.core.key import CKeyBase
 from bitcointx.signmessage import BitcoinMessage
 
 #Required only for PoDLE calculation:
@@ -60,7 +56,7 @@ def privkey_to_pubkey(priv):
     and return compressed/uncompressed public key as appropriate.'''
     compressed, priv = read_privkey(priv)
     #secp256k1 checks for validity of key value.
-    newpriv = secp256k1.PrivateKey(secret=native_bytes(priv))
+    newpriv = secp256k1.PrivateKey(secret=priv)
     return newpriv.public_key.format(compressed)
 
 # b58check wrapper functions around bitcointx.base58 functions:
@@ -137,7 +133,7 @@ def multiply(s, pub, return_serialized=True):
     '''
     newpub = secp256k1.PublicKey(pub)
     #see note to "tweak_mul" function in podle.py
-    res = newpub.multiply(native_bytes(s))
+    res = newpub.multiply(s)
     if not return_serialized:
         return res
     return res.format()
@@ -245,32 +241,9 @@ class JMCKey(bytes, CKeyBase):
     def __init__(self, b):
         CKeyBase.__init__(self, b, compressed=True)
 
-    def is_compressed(self):
-        return True
-
-    @property
-    def secret_bytes(self):
-        assert isinstance(self, bytes)
-        return self[:32]
-
     def sign(self, hash):
         assert isinstance(hash, (bytes, bytearray))
         if len(hash) != 32:
             raise ValueError('Hash must be exactly 32 bytes long')
         # TODO: non default sighash flag.
         return ecdsa_raw_sign(hash, self.secret_bytes + b"\x01", rawmsg=True)
-
-
-    def verify(self, hash, sig):
-        return self.pub.verify(hash, sig)
-
-    def verify_nonstrict(self, hash, sig):
-        return self.pub.verify_nonstrict(hash, sig)
-
-    @classmethod
-    def from_secret_bytes(cls, secret, compressed=True):
-        return cls(secret, compressed=compressed)
-
-    @classmethod
-    def from_bytes(cls, data):
-        raise NotImplementedError('subclasses must override from_bytes()')

jmbitcoin/jmbitcoin/secp256k1_transaction.py

@@ -1,20 +1,14 @@
 #!/usr/bin/python
-from past.builtins import basestring
-from io import BytesIO
-import binascii
-import copy
-import re
-import os
-import struct
+
 # note, only used for non-cryptographic randomness:
 import random
 from jmbitcoin.secp256k1_main import *
 
 from bitcointx.core import (CMutableTransaction, Hash160, CTxInWitness,
-                            CTxWitness, CMutableOutPoint, CMutableTxIn,
-                            CMutableTxOut, ValidationError, lx, x)
+                            CMutableOutPoint, CMutableTxIn,
+                            CMutableTxOut, ValidationError)
 from bitcointx.core.script import *
-from bitcointx.wallet import P2WPKHBitcoinAddress, CCoinAddress
+from bitcointx.wallet import P2WPKHCoinAddress, CCoinAddress, P2PKHCoinAddress
 from bitcointx.core.scripteval import (VerifyScript, SCRIPT_VERIFY_WITNESS,
                                        SCRIPT_VERIFY_P2SH, SIGVERSION_WITNESS_V0)
 
@@ -74,20 +68,15 @@ def pubkey_to_p2pkh_script(pub, require_compressed=False):
     representing the corresponding pay-to-pubkey-hash
     scriptPubKey.
     """
-    if not is_valid_pubkey(pub, require_compressed=require_compressed):
-        raise Exception("Invalid pubkey")
-    return CScript([OP_DUP, OP_HASH160, Hash160(pub),
-                                          OP_EQUALVERIFY, OP_CHECKSIG])
+    return P2PKHCoinAddress.from_pubkey(pub).to_scriptPubKey()
 
 def pubkey_to_p2wpkh_script(pub):
     """
     Given a pubkey in bytes (compressed), return a CScript
     representing the corresponding pay-to-witness-pubkey-hash
     scriptPubKey.
     """
-    if not is_valid_pubkey(pub, True):
-        raise Exception("Invalid pubkey")
-    return CScript([OP_0, Hash160(pub)])
+    return P2WPKHCoinAddress.from_pubkey(pub).to_scriptPubKey()
 
 def pubkey_to_p2sh_p2wpkh_script(pub):
     """
@@ -146,8 +135,7 @@ def return_err(e):
         return None, "Error in signing: " + repr(e)
 
     assert isinstance(tx, CMutableTransaction)
-    # using direct local access to libsecp256k1 binding, because
-    # python-bitcoinlib uses OpenSSL key management:
+
     pub = privkey_to_pubkey(priv)
 
     if not amount:
@@ -175,7 +163,7 @@ def return_err(e):
 
         input_scriptPubKey = pubkey_to_p2wpkh_script(pub)
         # only created for convenience access to scriptCode:
-        input_address = P2WPKHBitcoinAddress.from_scriptPubKey(input_scriptPubKey)
+        input_address = P2WPKHCoinAddress.from_scriptPubKey(input_scriptPubKey)
         # function name is misleading here; redeemScript only applies to p2sh.
         scriptCode = input_address.to_redeemScript()
 

jmbitcoin/jmbitcoin/snicker.py

@@ -1,7 +1,3 @@
-from __future__ import (absolute_import, division,
-                        print_function, unicode_literals)
-from builtins import * # noqa: F401
-
 # Implementation of proposal as per
 # https://gist.github.com/AdamISZ/2c13fb5819bd469ca318156e2cf25d79
 # (BIP SNICKER)
@@ -45,7 +41,7 @@ def verify_snicker_output(tx, pub, tweak, spk_type='p2sh-p2wpkh'):
     or -1 and None if it is not found exactly once.
     TODO Add support for other scriptPubKey types.
     """
-    assert isinstance(tx, btc.CBitcoinTransaction)
+    assert isinstance(tx, btc.CTransaction)
     expected_destination_pub = snicker_pubkey_tweak(pub, tweak)
     expected_destination_spk = pubkey_to_p2sh_p2wpkh_script(expected_destination_pub)
     found = 0

jmbitcoin/setup.py

@@ -9,6 +9,6 @@
       author_email='',
       license='GPL',
       packages=['jmbitcoin'],
-      install_requires=['future', 'coincurve', 'urldecode',
-                        'python-bitcointx>=1.0.5', 'pyaes'],
+      python_requires='>=3.6',
+      install_requires=['coincurve', 'python-bitcointx>=1.0.5', 'pyaes', 'urldecode'],
       zip_safe=False)

jmbitcoin/test/test_ecdh.py

@@ -1,7 +1,4 @@
 #! /usr/bin/env python
-from __future__ import (absolute_import, division,
-                        print_function, unicode_literals)
-from builtins import * # noqa: F401
 '''Tests coincurve binding to libsecp256k1 ecdh module code'''
 
 import hashlib

jmbitcoin/test/test_ecies.py

@@ -1,7 +1,4 @@
 #! /usr/bin/env python
-from __future__ import (absolute_import, division,
-                        print_function, unicode_literals)
-from builtins import * # noqa: F401
 '''Tests ECIES implementation as defined in BIP-SNICKER
 (and will be updated if that is).'''
 

jmbitcoin/test/test_tx_signing.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-import sys
 import pytest
 
 import binascii
@@ -26,7 +25,7 @@ def test_sign_standard_txs(addrtype):
     # (note that the input utxo is fake so we are really only creating
     # a destination here).
     scriptPubKey = btc.CScript([btc.OP_0, btc.Hash160(pub)])
-    address = btc.P2WPKHBitcoinAddress.from_scriptPubKey(scriptPubKey)
+    address = btc.P2WPKHCoinAddress.from_scriptPubKey(scriptPubKey)
 
     # Create a dummy outpoint; use same 32 bytes for convenience
     txid = priv[:32]
@@ -66,7 +65,7 @@ def test_mk_shuffled_tx():
     # prepare two addresses for the outputs
     pub = btc.privkey_to_pubkey(btc.Hash(b"priv") + b"\x01")
     scriptPubKey = btc.CScript([btc.OP_0, btc.Hash160(pub)])
-    addr1 = btc.P2WPKHBitcoinAddress.from_scriptPubKey(scriptPubKey)
+    addr1 = btc.P2WPKHCoinAddress.from_scriptPubKey(scriptPubKey)
     scriptPubKey_p2sh = scriptPubKey.to_p2sh_scriptPubKey()
     addr2 = btc.CCoinAddress.from_scriptPubKey(scriptPubKey_p2sh)
 

jmclient/jmclient/client_protocol.py

@@ -1,5 +1,4 @@
 #! /usr/bin/env python
-from future.utils import iteritems
 from twisted.internet import protocol, reactor, task
 from twisted.internet.error import (ConnectionLost, ConnectionAborted,
                                     ConnectionClosed, ConnectionDone)
@@ -15,7 +14,7 @@
 import os
 import sys
 from jmbase import (get_log, EXIT_FAILURE, hextobin, bintohex,
-                    utxo_to_utxostr, dictchanger)
+                    utxo_to_utxostr)
 from jmclient import (jm_single, get_irc_mchannels,
                       RegtestBitcoinCoreInterface)
 import jmbitcoin as btc
@@ -303,7 +302,7 @@ def on_JM_TX_RECEIVED(self, nick, txhex, offer):
         return {"accepted": True}
 
     def tx_match(self, txd):
-        for k,v in iteritems(self.finalized_offers):
+        for k,v in self.finalized_offers.items():
             # Tx considered defined by its output set
             if v["txd"].vout == txd.vout:
                 offerinfo = v

jmclient/jmclient/commitment_utils.py

@@ -1,10 +1,7 @@
 
 import sys
-import jmbitcoin as btc
 from jmbase import jmprint
-from jmclient import (jm_single, get_p2pk_vbyte, get_p2sh_vbyte,
-                      BTCEngine, TYPE_P2PKH, TYPE_P2SH_P2WPKH,
-                      BTC_P2PKH, BTC_P2SH_P2WPKH)
+from jmclient import jm_single, BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH
 from jmbase.support import EXIT_FAILURE, utxostr_to_utxo
 
 

jmclient/jmclient/configure.py

@@ -379,13 +379,13 @@ def validate_address(addr):
     try:
         # automatically respects the network
         # as set in btc.select_chain_params(...)
-        x = btc.CCoinAddress(addr)
+        dummyaddr = btc.CCoinAddress(addr)
     except Exception as e:
         return False, repr(e)
     # additional check necessary because python-bitcointx
     # does not check hash length on p2sh construction.
     try:
-        x.to_scriptPubKey()
+        dummyaddr.to_scriptPubKey()
     except Exception as e:
         return False, repr(e)
     return True, "address validated"