Merge pull request #1688 from theQRL/dev

Merged Dev to master

src/qrl/core/ChainManager.py

@@ -3,6 +3,7 @@
 # file LICENSE or http://www.opensource.org/licenses/mit-license.php.
 import sys
 import threading
+from collections import OrderedDict
 from typing import Optional, Tuple
 from math import ceil
 
@@ -102,24 +103,24 @@ def get_address_is_used(self, address: bytes) -> bool:
     def get_address_state(self, address: bytes) -> AddressState:
         """
         Transform Optimized Address State into Older Address State format
+        This should only be used by API.
         """
         optimized_address_state = self.get_optimized_address_state(address)
         ots_bitfield = [b'\x00'] * max(1024, int(ceil((2 ** optimized_address_state.height) / 8)))
         transaction_hashes = list()
-        tokens = dict()
-        slave_pks_access_type = dict()
+        tokens = OrderedDict()
+        slave_pks_access_type = OrderedDict()
 
         max_bitfield_page = ceil((2 ** optimized_address_state.height) / config.dev.ots_tracking_per_page)
 
-        offset = 0
         for page in range(1, max_bitfield_page + 1):
+            offset = (page - 1) * config.dev.ots_tracking_per_page
             page_data = self.get_bitfield(address, page)
             for data in page_data:
                 if offset >= len(ots_bitfield):
                     break
                 ots_bitfield[offset] = data
                 offset += 1
-            offset = (page - 1) * config.dev.ots_tracking_per_page
 
         max_transaction_hash_page = ceil(optimized_address_state.transaction_hash_count() / config.dev.data_per_page)
 
@@ -133,6 +134,11 @@ def get_address_state(self, address: bytes) -> AddressState:
             page_data = self.get_token_transaction_hashes(address, page * config.dev.data_per_page)
             for token_txn_hash in page_data:
                 token_balance = self.get_token(address, token_txn_hash)
+                # token_balance None is only possible when the token transaction
+                # is done by a QRL address as an owner, which has not been
+                # assigned any token balance.
+                if token_balance is None:
+                    continue
                 tokens[token_txn_hash] = token_balance.balance
 
         max_slave_page = ceil(optimized_address_state.slaves_count() / config.dev.data_per_page)

src/qrl/core/OptimizedAddressState.py

@@ -41,6 +41,10 @@ def __init__(self, protobuf_block=None):
     def bin_to_qaddress(binAddress):
         return 'Q' + bin2hstr(binAddress)
 
+    @staticmethod
+    def get_height_from_address(bin_address):
+        return bin_address[1] << 1
+
     @property
     def pbdata(self):
         """

src/qrl/core/VoteStats.py

@@ -233,7 +233,7 @@ def revert(self,
             addr_to = multi_sig_spend.addrs_to[index]
             address_state = addresses_state[addr_to]
             if addr_to not in (multi_sig_spend.addr_from, addr_from_pk):
-                paginated_tx_hash.insert(address_state, multi_sig_spend.txhash)
+                paginated_tx_hash.remove(address_state, multi_sig_spend.txhash)
             address_state.update_balance(state_container, multi_sig_spend.amounts[index], subtract=True)
 
         self._data.executed = False

src/qrl/core/config.py

@@ -305,7 +305,7 @@ def __init__(self, pbdata, ignore_check=False, ignore_singleton=False):
         # ======================================
         # HARD FORK HEIGHTS LIST
         # ======================================
-        self.hard_fork_heights = [1000000]
+        self.hard_fork_heights = [942375]
         self.testnet_hard_fork_heights = [10500]
 
         # ======================================

src/qrl/core/p2p/p2pPeerManager.py

@@ -67,7 +67,11 @@ def trusted_peer(self, channel: P2PProtocol):
 
     @property
     def trusted_addresses(self):
-        return set([peer.peer.full_address for peer in self._p2p_factory.connections if self.trusted_peer(peer)])
+        ip_public_port_set = set()
+        for peer in self._p2p_factory.connections:
+            if self.trusted_peer(peer) and peer.public_port != 0:
+                ip_public_port_set.add(peer.ip_public_port)
+        return ip_public_port_set
 
     @property
     def peer_node_status(self):
@@ -195,7 +199,8 @@ def _get_version_compatibility(self, version) -> bool:
                 if int(major_version) < 2:
                     return False
             except Exception:
-                return False
+                logger.warning("Exception while checking version for compatibility")
+                return True
 
         return True
 
@@ -228,6 +233,7 @@ def handle_version(self, source, message: qrllegacy_pb2.LegacyMessage):
                            source.peer.ip,
                            message.veData.version)
             source.loseConnection()
+            self.ban_channel(source)
             return
 
         source.rate_limit = min(config.user.peer_rate_limit, message.veData.rate_limit)
@@ -237,6 +243,7 @@ def handle_version(self, source, message: qrllegacy_pb2.LegacyMessage):
             logger.warning('Expected: %s', config.user.genesis_prev_headerhash)
             logger.warning('Found: %s', message.veData.genesis_prev_hash)
             source.loseConnection()
+            self.ban_channel(source)
 
     def handle_peer_list(self, source, message: qrllegacy_pb2.LegacyMessage):
         P2PBaseObserver._validate_message(message, qrllegacy_pb2.LegacyMessage.PL)

src/qrl/core/p2p/p2pfactory.py

@@ -230,6 +230,7 @@ def block_received(self, source, block: Block):
 
         if block.block_number != self._last_requested_block_number:
             logger.warning('Did not match %s', self._last_requested_block_number)
+            self._qrl_node.peer_manager.ban_channel(source)
             return
 
         target_start_blocknumber = self._target_node_header_hash.block_number
@@ -238,17 +239,20 @@ def block_received(self, source, block: Block):
             logger.warning('Did not match headerhash')
             logger.warning('Expected headerhash %s', expected_headerhash)
             logger.warning('Found headerhash %s', block.headerhash)
+            self._qrl_node.peer_manager.ban_channel(source)
             return
 
         if not block.validate(self._chain_manager, self.pow.future_blocks):
             logger.warning('Syncing Failed: Block Validation Failed')
+            self._qrl_node.peer_manager.ban_channel(source)
             return
 
         if self._chain_manager.add_block(block, check_stale=False):
             if self._chain_manager.last_block.headerhash == block.headerhash:
                 self.pow.suspend_mining_timestamp = ntp.getTime() + config.dev.sync_delay_mining
         else:
             logger.warning('Failed to Add Block')
+            self._qrl_node.peer_manager.ban_channel(source)
             return
 
         try:

src/qrl/core/qrlnode.py

@@ -401,8 +401,16 @@ def get_ots(self,
                 page_from: int,
                 page_count: int,
                 unused_ots_index_from: int) -> (list, Optional[int], bool):
+        if not OptimizedAddressState.address_is_valid(address):
+            return None, None, None
+
+        max_bitfield = 2 ** OptimizedAddressState.get_height_from_address(address)
+        max_pages = (max_bitfield // config.dev.ots_tracking_per_page) + 1
+        page_from = min(page_from, max_pages)
+        max_pages = min(page_from + page_count, max_pages)
+
         bitfields = list()
-        for page in range(page_from, page_from + page_count):
+        for page in range(page_from, max_pages):
             bitfield = self._chain_manager.get_bitfield(address, page)
             bitfields.append(qrl_pb2.OTSBitfieldByPage(ots_bitfield=bitfield, page_number=page))
 
@@ -564,6 +572,13 @@ def get_mini_transactions_by_address(self, address: bytes, item_per_page: int, p
             elif isinstance(tx, CoinBase):
                 if tx.addr_to == address:
                     amount += tx.amount
+            elif isinstance(tx, MultiSigSpend):
+                try:
+                    for i in range(len(tx.addrs_to)):
+                        if tx.addrs_to[i] == address:
+                            amount += tx.amounts[i]
+                except ValueError:
+                    pass
 
             if amount < 0:
                 mini_transaction.out = True

src/qrl/core/txs/LatticeTransaction.py

@@ -113,9 +113,10 @@ def revert(self,
         address_state = state_container.addresses_state[self.addr_from]
         address_state.update_balance(state_container, self.fee)
         state_container.paginated_lattice_pk.remove(address_state, self.txhash)
-        state_container.paginated_tx_hash.remmove(address_state, self.txhash)
+        state_container.paginated_tx_hash.remove(address_state, self.txhash)
 
         state_container.lattice_pk.data[(self.addr_from,
-                                         self.pk1, self.pk2, self.pk3)] = LatticePKMetadata(enabled=False)
+                                         self.pk1, self.pk2, self.pk3)] = LatticePKMetadata(enabled=False,
+                                                                                            delete=True)
 
         return self._revert_state_changes_for_PK(state_container)

src/qrl/core/txs/TokenTransaction.py

@@ -97,21 +97,14 @@ def _validate_custom(self):
             logger.warning('Token decimals cannot be more than 19')
             return False
 
-        addresses = set()
         sum_of_initial_balances = 0
         for initial_balance in self.initial_balances:
             sum_of_initial_balances += initial_balance.amount
-            addresses.add(initial_balance.address)  # TODO: Hard fork code, check if compatible with older blocks
             if initial_balance.amount <= 0:
                 logger.warning('Invalid Initial Amount in Token Transaction')
                 logger.warning('Address %s | Amount %s', initial_balance.address, initial_balance.amount)
                 return False
 
-        # TODO: Hard fork code, check if compatible with older blocks
-        if len(addresses) != len(self.initial_balances):
-            logger.warning('Invalid Token Initialization. Duplicate address found initial_balance')
-            return False
-
         allowed_decimals = self.calc_allowed_decimals(sum_of_initial_balances // 10 ** self.decimals)
 
         if self.decimals > allowed_decimals:
@@ -183,10 +176,18 @@ def apply(self,
             if initial_balance.address == addr_from_pk:
                 addr_from_pk_processed = True
 
-            state_container.tokens.data[(initial_balance.address,
-                                         self.txhash)] = TokenBalance(balance=initial_balance.amount,
-                                                                      decimals=self.decimals,
-                                                                      delete=False)
+            # If a QRL address has been mentioned multiple times in initial balance
+            # then check if that address has already been initialized with some token
+            # balance, if found, then add the new balance the already initialized balance
+            if (initial_balance.address, self.txhash) in state_container.tokens.data:
+                state_container.tokens.data[(initial_balance.address,
+                                             self.txhash)].balance += initial_balance.amount
+            else:
+                state_container.tokens.data[(initial_balance.address,
+                                             self.txhash)] = TokenBalance(balance=initial_balance.amount,
+                                                                          decimals=self.decimals,
+                                                                          tx_hash=self.txhash,
+                                                                          delete=False)
             address_state = state_container.addresses_state[initial_balance.address]
             state_container.paginated_tx_hash.insert(address_state, self.txhash)
             state_container.paginated_tokens_hash.insert(address_state, self.txhash)

src/qrl/core/txs/TransferTokenTransaction.py

@@ -169,8 +169,9 @@ def apply(self,
                 state_container.tokens.data[(addr_to,
                                              self.token_txhash)] = TokenBalance(balance=0,
                                                                                 decimals=decimals,
+                                                                                tx_hash=self.txhash,
                                                                                 delete=False)
-                state_container.paginated_tokens_hash.insert(address_state, self.txhash)
+                state_container.paginated_tokens_hash.insert(address_state, self.token_txhash)
 
             state_container.tokens.data[(addr_to, self.token_txhash)].balance += amount
 
@@ -182,19 +183,28 @@ def apply(self,
     def revert(self,
                state: State,
                state_container: StateContainer) -> bool:
-        state_container.tokens.data[(self.addr_from, self.token_txhash)].balance += self.total_amount
-        address_state = state_container.addresses_state[self.addr_from]
-        address_state.update_balance(state_container, self.fee)
-        state_container.paginated_tx_hash.remove(address_state, self.txhash)
-
         for index in range(0, len(self.addrs_to)):
             addr_to = self.addrs_to[index]
             amount = self.amounts[index]
             address_state = state_container.addresses_state[addr_to]
+            key = (addr_to, self.token_txhash)
 
-            state_container.tokens.data[(addr_to, self.token_txhash)].balance -= amount
+            state_container.tokens.data[key].balance -= amount
+            # There is a chance that same address is transmitted with token multiple times,
+            # in such a case, to avoid removal of token_txhash from paginated_tokens_hash
+            # delete must be checked for false
+            if state_container.tokens.data[key].tx_hash == self.txhash and \
+                    state_container.tokens.data[key].delete is False:
+                state_container.tokens.data[key].delete = True
+                state_container.paginated_tokens_hash.remove(address_state, self.token_txhash)
 
             if self.addr_from != addr_to:
                 state_container.paginated_tx_hash.remove(address_state, self.txhash)
 
+        state_container.tokens.data[(self.addr_from, self.token_txhash)].balance += self.total_amount
+        state_container.tokens.data[(self.addr_from, self.token_txhash)].delete = False
+        address_state = state_container.addresses_state[self.addr_from]
+        address_state.update_balance(state_container, self.fee)
+        state_container.paginated_tx_hash.remove(address_state, self.txhash)
+
         return self._revert_state_changes_for_PK(state_container)

src/qrl/core/txs/multisig/MultiSigVote.py

@@ -74,6 +74,9 @@ def _validate_extended(self, state_container: StateContainer):
 
         addr_from_state = state_container.addresses_state[self.addr_from]
         vote_stats = state_container.votes_stats[self.shared_key]
+        if vote_stats is None:
+            logger.warning("[MultiSigVote] Invalid Shared key %s", bin2hstr(self.shared_key))
+            return False
         multi_sig_spend_tx = state_container.multi_sig_spend_txs[self.shared_key]
         block_number = state_container.block_number
         if vote_stats.executed:
@@ -143,9 +146,6 @@ def apply(self,
     def revert(self,
                state: State,
                state_container: StateContainer) -> bool:
-        address_state = state_container.addresses_state[self.addr_from]
-        address_state.update_balance(state_container, self.fee)
-        state_container.paginated_tx_hash.remove(address_state, self.txhash)
         vote_stats = state_container.votes_stats[self.shared_key]
         multi_sig_address = vote_stats.multi_sig_address
 
@@ -158,4 +158,8 @@ def revert(self,
             logger.info("[MultiSigVote] Failed to revert vote_stats")
             return False
 
+        address_state = state_container.addresses_state[self.addr_from]
+        address_state.update_balance(state_container, self.fee)
+        state_container.paginated_tx_hash.remove(address_state, self.txhash)
+
         return self._revert_state_changes_for_PK(state_container)